Merge branch 'develop' of github.com:Budibase/budibase into feature/BUDI-7052

.github/workflows/budibase_ci.yml

@@ -12,9 +12,6 @@ on:
       - master
       - develop
   pull_request:
-    branches:
-      - master
-      - develop
   workflow_dispatch:
 
 env:

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 14.20.1
+nodejs 14.21.3
 python 3.10.0

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.12-alpha.0",
+  "version": "2.8.12-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -53,7 +53,7 @@
     "kill-all": "yarn run kill-builder && yarn run kill-server",
     "dev": "yarn run kill-all && lerna run --stream dev:builder --stream",
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
+    "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/backend-core/src/auth/auth.ts

@@ -159,7 +159,7 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
 
   try {
     const db = getGlobalDB()
-    const dbUser = await db.get(userId)
+    const dbUser = await db.get<any>(userId)
 
     //Do not overwrite the refresh token if a valid one is not provided.
     if (typeof details.refreshToken !== "string") {

packages/backend-core/src/cache/user.ts

@@ -12,7 +12,7 @@ const EXPIRY_SECONDS = 3600
  */
 async function populateFromDB(userId: string, tenantId: string) {
   const db = tenancy.getTenantDB(tenantId)
-  const user = await db.get(userId)
+  const user = await db.get<any>(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)

packages/backend-core/src/db/searchIndexes/searchIndexes.ts

@@ -5,7 +5,7 @@ export async function createUserIndex() {
   const db = getGlobalDB()
   let designDoc
   try {
-    designDoc = await db.get("_design/database")
+    designDoc = await db.get<any>("_design/database")
   } catch (err: any) {
     if (err.status === 404) {
       designDoc = { _id: "_design/database" }

packages/backend-core/src/users.ts

@@ -67,9 +67,9 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
 
 export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
-  let user = await db.get(id)
+  let user = await db.get<User>(id)
   if (opts?.cleanup) {
-    user = removeUserPassword(user)
+    user = removeUserPassword(user) as User
   }
   return user
 }

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -35,9 +35,8 @@
     try {
       const isSelected =
         decodeURIComponent($params.viewName) === $views.selectedViewName
-      const name = view.name
       const id = view.tableId
-      await views.delete(name)
+      await views.delete(view)
       notifications.success("View deleted")
       if (isSelected) {
         $goto(`./table/${id}`)

packages/builder/src/components/common/NavItem.svelte

@@ -2,7 +2,7 @@
   import { Icon } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
-  import UserAvatars from "../../pages/builder/app/[application]/_components/UserAvatars.svelte"
+  import { UserAvatars } from "@budibase/frontend-core"
 
   export let icon
   export let withArrow = false

packages/builder/src/components/start/AppRow.svelte

@@ -2,12 +2,12 @@
   import { Heading, Body, Button, Icon } from "@budibase/bbui"
   import { processStringSync } from "@budibase/string-templates"
   import { goto } from "@roxi/routify"
-  import { UserAvatar } from "@budibase/frontend-core"
+  import { UserAvatars } from "@budibase/frontend-core"
 
   export let app
   export let lockedAction
 
-  $: editing = app?.lockedBy != null
+  $: editing = app.sessions?.length
 
   const handleDefaultClick = () => {
     if (window.innerWidth < 640) {
@@ -41,7 +41,7 @@
   <div class="updated">
     {#if editing}
       Currently editing
-      <UserAvatar user={app.lockedBy} />
+      <UserAvatars users={app.sessions} />
     {:else if app.updatedAt}
       {processStringSync("Updated {{ duration time 'millisecond' }} ago", {
         time: new Date().getTime() - new Date(app.updatedAt).getTime(),

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -26,7 +26,7 @@
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import TourPopover from "components/portal/onboarding/TourPopover.svelte"
   import BuilderSidePanel from "./_components/BuilderSidePanel.svelte"
-  import UserAvatars from "./_components/UserAvatars.svelte"
+  import { UserAvatars } from "@budibase/frontend-core"
   import { TOUR_KEYS, TOURS } from "components/portal/onboarding/tours.js"
   import PreviewOverlay from "./_components/PreviewOverlay.svelte"
 

packages/builder/src/stores/backend/views.js

@@ -26,14 +26,12 @@ export function createViewsStore() {
   }
 
   const deleteView = async view => {
-    await API.deleteView(view)
+    await API.deleteView(view.name)
 
     // Update tables
     tables.update(state => {
       const table = state.list.find(table => table._id === view.tableId)
-      if (table) {
+      delete table.views[view.name]
-        delete table.views[view.name]
-      }
       return { ...state }
     })
   }

packages/frontend-core/src/components/index.js

@@ -2,4 +2,5 @@ export { default as SplitPage } from "./SplitPage.svelte"
 export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
+export { default as UserAvatars } from "./UserAvatars.svelte"
 export { Grid } from "./grid"

packages/pro

@@ -1 +1 @@
-Subproject commit 544c7e067de69832469cde673e59501480d6d98a
+Subproject commit 9c564edb37cb619cb5971e10c4317fa6e7c5bb00

packages/server/src/api/controllers/analytics.ts

@@ -1,5 +1,5 @@
 import { events } from "@budibase/backend-core"
-import { AnalyticsPingRequest, PingSource } from "@budibase/types"
+import { AnalyticsPingRequest, App, PingSource } from "@budibase/types"
 import { DocumentType, isDevAppID } from "../../db/utils"
 import { context } from "@budibase/backend-core"
 
@@ -16,7 +16,7 @@ export const ping = async (ctx: any) => {
   switch (body.source) {
     case PingSource.APP: {
       const db = context.getAppDB({ skip_setup: true })
-      const appInfo = await db.get(DocumentType.APP_METADATA)
+      const appInfo = await db.get<App>(DocumentType.APP_METADATA)
       let appId = context.getAppId()
 
       if (isDevAppID(appId)) {

packages/server/src/api/controllers/apikeys.ts

@@ -6,7 +6,7 @@ const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys
 async function getBuilderMainDoc() {
   const db = tenancy.getGlobalDB()
   try {
-    return await db.get(KEYS_DOC)
+    return await db.get<any>(KEYS_DOC)
   } catch (err) {
     // doesn't exist yet, nothing to get
     return {

packages/server/src/api/controllers/application.ts

@@ -49,6 +49,7 @@ import {
   MigrationType,
   PlanType,
   Screen,
+  SocketSession,
   UserCtx,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
@@ -183,6 +184,7 @@ export async function fetch(ctx: UserCtx) {
   const appIds = apps
     .filter((app: any) => app.status === "development")
     .map((app: any) => app.appId)
+
   // get the locks for all the dev apps
   if (dev || all) {
     const locks = await getLocksById(appIds)
@@ -197,7 +199,10 @@ export async function fetch(ctx: UserCtx) {
     }
   }
 
-  ctx.body = await checkAppMetadata(apps)
+  // Enrich apps with all builder user sessions
+  const enrichedApps = await sdk.users.sessions.enrichApps(apps)
+
+  ctx.body = await checkAppMetadata(enrichedApps)
 }
 
 export async function fetchAppDefinition(ctx: UserCtx) {
@@ -217,7 +222,7 @@ export async function fetchAppDefinition(ctx: UserCtx) {
 
 export async function fetchAppPackage(ctx: UserCtx) {
   const db = context.getAppDB()
-  let application = await db.get(DocumentType.APP_METADATA)
+  let application = await db.get<any>(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
   const license = await licensing.cache.getCachedLicense()
@@ -453,7 +458,7 @@ export async function update(ctx: UserCtx) {
 export async function updateClient(ctx: UserCtx) {
   // Get current app version
   const db = context.getAppDB()
-  const application = await db.get(DocumentType.APP_METADATA)
+  const application = await db.get<App>(DocumentType.APP_METADATA)
   const currentVersion = application.version
 
   // Update client library and manifest
@@ -477,7 +482,7 @@ export async function updateClient(ctx: UserCtx) {
 export async function revertClient(ctx: UserCtx) {
   // Check app can be reverted
   const db = context.getAppDB()
-  const application = await db.get(DocumentType.APP_METADATA)
+  const application = await db.get<App>(DocumentType.APP_METADATA)
   if (!application.revertableVersion) {
     ctx.throw(400, "There is no version to revert to")
   }
@@ -530,7 +535,7 @@ async function destroyApp(ctx: UserCtx) {
 
   const db = dbCore.getDB(devAppId)
   // standard app deletion flow
-  const app = await db.get(DocumentType.APP_METADATA)
+  const app = await db.get<App>(DocumentType.APP_METADATA)
   const result = await db.destroy()
   await quotas.removeApp()
   await events.app.deleted(app)
@@ -593,7 +598,7 @@ export async function sync(ctx: UserCtx) {
 export async function updateAppPackage(appPackage: any, appId: any) {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
-    const application = await db.get(DocumentType.APP_METADATA)
+    const application = await db.get<App>(DocumentType.APP_METADATA)
 
     const newAppPackage = { ...application, ...appPackage }
     if (appPackage._rev !== application._rev) {

packages/server/src/api/controllers/auth.ts

@@ -4,6 +4,7 @@ import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
 import { ContextUser, User, Row, UserCtx } from "@budibase/types"
+import sdk from "../../sdk"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -41,7 +42,7 @@ export async function fetchSelf(ctx: UserCtx) {
     // remove the full roles structure
     delete user.roles
     try {
-      const userTable = await db.get(InternalTables.USER_METADATA)
+      const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA)
       // specifically needs to make sure is enriched
       ctx.body = await outputProcessing(userTable, user as Row)
     } catch (err: any) {

packages/server/src/api/controllers/automation.ts

@@ -16,6 +16,7 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
 import { automations, features } from "@budibase/pro"
 import {
+  App,
   Automation,
   AutomationActionStepId,
   AutomationResults,
@@ -152,7 +153,7 @@ export async function update(ctx: BBContext) {
     return
   }
 
-  const oldAutomation = await db.get(automation._id)
+  const oldAutomation = await db.get<Automation>(automation._id)
   automation = cleanAutomationInputs(automation)
   automation = await checkForWebhooks({
     oldAuto: oldAutomation,
@@ -210,7 +211,7 @@ export async function find(ctx: BBContext) {
 export async function destroy(ctx: BBContext) {
   const db = context.getAppDB()
   const automationId = ctx.params.id
-  const oldAutomation = await db.get(automationId)
+  const oldAutomation = await db.get<Automation>(automationId)
   await checkForWebhooks({
     oldAuto: oldAutomation,
   })
@@ -229,7 +230,7 @@ export async function clearLogError(ctx: BBContext) {
   const { automationId, appId } = ctx.request.body
   await context.doInAppContext(appId, async () => {
     const db = context.getProdAppDB()
-    const metadata = await db.get(DocumentType.APP_METADATA)
+    const metadata = await db.get<App>(DocumentType.APP_METADATA)
     if (!automationId) {
       delete metadata.automationErrors
     } else if (
@@ -267,7 +268,7 @@ export async function getDefinitionList(ctx: BBContext) {
 
 export async function trigger(ctx: BBContext) {
   const db = context.getAppDB()
-  let automation = await db.get(ctx.params.id)
+  let automation = await db.get<Automation>(ctx.params.id)
 
   let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation)
   if (hasCollectStep && (await features.isSyncAutomationsEnabled())) {
@@ -312,8 +313,8 @@ function prepareTestInput(input: any) {
 
 export async function test(ctx: BBContext) {
   const db = context.getAppDB()
-  let automation = await db.get(ctx.params.id)
+  let automation = await db.get<Automation>(ctx.params.id)
-  await setTestFlag(automation._id)
+  await setTestFlag(automation._id!)
   const testInput = prepareTestInput(ctx.request.body)
   const response = await triggers.externalTrigger(
     automation,
@@ -328,7 +329,7 @@ export async function test(ctx: BBContext) {
     ...ctx.request.body,
     occurredAt: new Date().getTime(),
   })
-  await clearTestFlag(automation._id)
+  await clearTestFlag(automation._id!)
   ctx.body = response
   await events.automation.tested(automation)
 }

packages/server/src/api/controllers/backup.ts

@@ -1,7 +1,7 @@
 import sdk from "../../sdk"
 import { events, context, db } from "@budibase/backend-core"
 import { DocumentType } from "../../db/utils"
-import { Ctx } from "@budibase/types"
+import { App, Ctx } from "@budibase/types"
 
 interface ExportAppDumpRequest {
   excludeRows: boolean
@@ -29,7 +29,7 @@ export async function exportAppDump(ctx: Ctx<ExportAppDumpRequest>) {
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()
-    const app = await appDb.get(DocumentType.APP_METADATA)
+    const app = await appDb.get<App>(DocumentType.APP_METADATA)
     await events.app.exported(app)
   })
 }

packages/server/src/api/controllers/component.ts

@@ -1,5 +1,5 @@
 import { DocumentType } from "../../db/utils"
-import { Plugin } from "@budibase/types"
+import { App, Plugin } from "@budibase/types"
 import { db as dbCore, context, tenancy } from "@budibase/backend-core"
 import { getComponentLibraryManifest } from "../../utilities/fileSystem"
 import { UserCtx } from "@budibase/types"
@@ -7,7 +7,7 @@ import { UserCtx } from "@budibase/types"
 export async function fetchAppComponentDefinitions(ctx: UserCtx) {
   try {
     const db = context.getAppDB()
-    const app = await db.get(DocumentType.APP_METADATA)
+    const app = await db.get<App>(DocumentType.APP_METADATA)
 
     let componentManifests = await Promise.all(
       app.componentLibraries.map(async (library: any) => {

packages/server/src/api/controllers/datasource.ts

@@ -9,7 +9,6 @@ import {
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
-import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
 import {
@@ -433,8 +432,7 @@ export async function destroy(ctx: UserCtx) {
 }
 
 export async function find(ctx: UserCtx) {
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const datasource = await db.get(ctx.params.datasourceId)
   ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
@@ -442,15 +440,14 @@ export async function find(ctx: UserCtx) {
 export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await getDatasourceAndQuery(queryJson)
+    ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
   } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
 export async function getExternalSchema(ctx: UserCtx) {
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const datasource = await db.get(ctx.params.datasourceId)
   const enrichedDatasource = await getAndMergeDatasource(datasource)
   const connector = await getConnector(enrichedDatasource)
 

packages/server/src/api/controllers/deploy/index.ts

@@ -7,7 +7,7 @@ import {
   enableCronTrigger,
 } from "../../../automations/utils"
 import { backups } from "@budibase/pro"
-import { AppBackupTrigger } from "@budibase/types"
+import { App, AppBackupTrigger } from "@budibase/types"
 import sdk from "../../../sdk"
 import { builderSocket } from "../../../websockets"
 
@@ -44,7 +44,7 @@ async function storeDeploymentHistory(deployment: any) {
   let deploymentDoc
   try {
     // theres only one deployment doc per app database
-    deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
+    deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
   } catch (err) {
     deploymentDoc = { _id: DocumentType.DEPLOYMENTS, history: {} }
   }
@@ -113,7 +113,7 @@ export async function fetchDeployments(ctx: any) {
 export async function deploymentProgress(ctx: any) {
   try {
     const db = context.getAppDB()
-    const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
+    const deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
     ctx.body = deploymentDoc[ctx.params.deploymentId]
   } catch (err) {
     ctx.throw(
@@ -165,9 +165,9 @@ export const publishApp = async function (ctx: any) {
     // app metadata is excluded as it is likely to be in conflict
     // replicate the app metadata document manually
     const db = context.getProdAppDB()
-    const appDoc = await devDb.get(DocumentType.APP_METADATA)
+    const appDoc = await devDb.get<App>(DocumentType.APP_METADATA)
     try {
-      const prodAppDoc = await db.get(DocumentType.APP_METADATA)
+      const prodAppDoc = await db.get<App>(DocumentType.APP_METADATA)
       appDoc._rev = prodAppDoc._rev
     } catch (err) {
       delete appDoc._rev

packages/server/src/api/controllers/dev.ts

@@ -6,6 +6,7 @@ import { clearLock as redisClearLock } from "../../utilities/redis"
 import { DocumentType } from "../../db/utils"
 import { context, env as envCore } from "@budibase/backend-core"
 import { events, db as dbCore, cache } from "@budibase/backend-core"
+import { App } from "@budibase/types"
 
 async function redirect(ctx: any, method: string, path: string = "global") {
   const { devPath } = ctx.params
@@ -81,7 +82,7 @@ export async function revert(ctx: any) {
     if (!exists) {
       throw new Error("App must be deployed to be reverted.")
     }
-    const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
+    const deploymentDoc = await db.get<any>(DocumentType.DEPLOYMENTS)
     if (
       !deploymentDoc.history ||
       Object.keys(deploymentDoc.history).length === 0
@@ -104,7 +105,7 @@ export async function revert(ctx: any) {
 
     // update appID in reverted app to be dev version again
     const db = context.getAppDB()
-    const appDoc = await db.get(DocumentType.APP_METADATA)
+    const appDoc = await db.get<App>(DocumentType.APP_METADATA)
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)

packages/server/src/api/controllers/public/utils.ts

@@ -14,7 +14,7 @@ export async function addRev(
     id = DocumentType.APP_METADATA
   }
   const db = context.getAppDB()
-  const dbDoc = await db.get(id)
+  const dbDoc = await db.get<any>(id)
   body._rev = dbDoc._rev
   // update ID in case it is an app ID
   body._id = id

packages/server/src/api/controllers/query/index.ts

@@ -9,6 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
+import { Query } from "@budibase/types"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@@ -206,7 +207,7 @@ async function execute(
 ) {
   const db = context.getAppDB()
 
-  const query = await db.get(ctx.params.queryId)
+  const query = await db.get<Query>(ctx.params.queryId)
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     query.datasourceId
   )
@@ -275,7 +276,7 @@ export async function executeV2(
 
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
-  const query = await db.get(queryId)
+  const query = await db.get<Query>(queryId)
   const datasource = await sdk.datasources.get(query.datasourceId)
   const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
@@ -298,7 +299,7 @@ export async function destroy(ctx: any) {
   const db = context.getAppDB()
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
-  const query = await db.get(queryId)
+  const query = await db.get<Query>(queryId)
   const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`

packages/server/src/api/controllers/role.ts

@@ -1,6 +1,7 @@
 import { roles, context, events, db as dbCore } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { UserCtx, Database } from "@budibase/types"
+import { UserCtx, Database, UserRoles, Role } from "@budibase/types"
+import sdk from "../../sdk"
 
 const UpdateRolesOptions = {
   CREATED: "created",
@@ -13,23 +14,23 @@ async function updateRolesOnUserTable(
   updateOption: string,
   roleVersion: string | undefined
 ) {
-  const table = await db.get(InternalTables.USER_METADATA)
+  const table = await sdk.tables.getTable(InternalTables.USER_METADATA)
   const schema = table.schema
   const remove = updateOption === UpdateRolesOptions.REMOVED
   let updated = false
   for (let prop of Object.keys(schema)) {
     if (prop === "roleId") {
       updated = true
-      const constraints = schema[prop].constraints
+      const constraints = schema[prop].constraints!
       const updatedRoleId =
         roleVersion === roles.RoleIDVersion.NAME
           ? roles.getExternalRoleID(roleId, roleVersion)
           : roleId
-      const indexOfRoleId = constraints.inclusion.indexOf(updatedRoleId)
+      const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
       if (remove && indexOfRoleId !== -1) {
-        constraints.inclusion.splice(indexOfRoleId, 1)
+        constraints.inclusion!.splice(indexOfRoleId, 1)
       } else if (!remove && indexOfRoleId === -1) {
-        constraints.inclusion.push(updatedRoleId)
+        constraints.inclusion!.push(updatedRoleId)
       }
       break
     }
@@ -69,7 +70,7 @@ export async function save(ctx: UserCtx) {
 
   let dbRole
   if (!isCreate) {
-    dbRole = await db.get(_id)
+    dbRole = await db.get<UserRoles>(_id)
   }
   if (dbRole && dbRole.name !== name && isNewVersion) {
     ctx.throw(400, "Cannot change custom role name")
@@ -105,7 +106,7 @@ export async function destroy(ctx: UserCtx) {
     // make sure has the prefix (if it has it then it won't be added)
     roleId = dbCore.generateRoleID(roleId)
   }
-  const role = await db.get(roleId)
+  const role = await db.get<Role>(roleId)
   // first check no users actively attached to role
   const users = (
     await db.allDocs(

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -23,14 +23,13 @@ import {
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
-import { getDatasourceAndQuery } from "./utils"
+import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { isEditableColumn } from "../../../sdk/app/tables/validation"
 
 export interface ManyRelationship {
   tableId?: string

packages/server/src/api/controllers/row/external.ts

@@ -1,30 +1,20 @@
-import {
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-  FieldTypes,
-  NoEmptyFilterStrings,
-  SortDirection,
-} from "../../../constants"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../integrations/utils"
 import { ExternalRequest, RunConfig } from "./ExternalRequest"
-import * as exporters from "../view/exporters"
-import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Datasource,
   IncludeRelationship,
   Operation,
-  PaginationJson,
   Row,
-  SortJson,
   Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 
-const { cleanExportRows } = require("./utils")
-
 async function getRow(
   tableId: string,
   rowId: string,
@@ -59,6 +49,7 @@ export async function handleRequest(
       }
     }
   }
+
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
     opts || {}
   )
@@ -114,21 +105,6 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function fetchView(ctx: UserCtx) {
-  // there are no views in external datasources, shouldn't ever be called
-  // for now just fetch
-  const split = ctx.params.viewName.split("all_")
-  ctx.params.tableId = split[1] ? split[1] : split[0]
-  return fetch(ctx)
-}
-
-export async function fetch(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  return handleRequest(Operation.READ, tableId, {
-    includeSqlRelationships: IncludeRelationship.INCLUDE,
-  })
-}
-
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
@@ -161,129 +137,6 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  const { paginate, query, ...params } = ctx.request.body
-  let { bookmark, limit } = params
-  if (!bookmark && paginate) {
-    bookmark = 1
-  }
-  let paginateObj = {}
-
-  if (paginate) {
-    paginateObj = {
-      // add one so we can track if there is another page
-      limit: limit,
-      page: bookmark,
-    }
-  } else if (params && limit) {
-    paginateObj = {
-      limit: limit,
-    }
-  }
-  let sort: SortJson | undefined
-  if (params.sort) {
-    const direction =
-      params.sortOrder === "descending"
-        ? SortDirection.DESCENDING
-        : SortDirection.ASCENDING
-    sort = {
-      [params.sort]: { direction },
-    }
-  }
-  try {
-    const rows = (await handleRequest(Operation.READ, tableId, {
-      filters: query,
-      sort,
-      paginate: paginateObj as PaginationJson,
-      includeSqlRelationships: IncludeRelationship.INCLUDE,
-    })) as Row[]
-    let hasNextPage = false
-    if (paginate && rows.length === limit) {
-      const nextRows = (await handleRequest(Operation.READ, tableId, {
-        filters: query,
-        sort,
-        paginate: {
-          limit: 1,
-          page: bookmark * limit + 1,
-        },
-        includeSqlRelationships: IncludeRelationship.INCLUDE,
-      })) as Row[]
-      hasNextPage = nextRows.length > 0
-    }
-    // need wrapper object for bookmarks etc when paginating
-    return { rows, hasNextPage, bookmark: bookmark + 1 }
-  } catch (err: any) {
-    if (err.message && err.message.includes("does not exist")) {
-      throw new Error(
-        `Table updated externally, please re-fetch - ${err.message}`
-      )
-    } else {
-      throw err
-    }
-  }
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
-  const format = ctx.query.format
-  const { columns } = ctx.request.body
-  const datasource = await sdk.datasources.get(datasourceId!)
-  if (!datasource || !datasource.entities) {
-    ctx.throw(400, "Datasource has not been configured for plus API.")
-  }
-
-  if (ctx.request.body.rows) {
-    ctx.request.body = {
-      query: {
-        oneOf: {
-          _id: ctx.request.body.rows.map((row: string) => {
-            const ids = JSON.parse(
-              decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
-            )
-            if (ids.length > 1) {
-              ctx.throw(400, "Export data does not support composite keys.")
-            }
-            return ids[0]
-          }),
-        },
-      },
-    }
-  }
-
-  let result = await search(ctx)
-  let rows: Row[] = []
-
-  // Filter data to only specified columns if required
-
-  if (columns && columns.length) {
-    for (let i = 0; i < result.rows.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result.rows[i][column]
-      }
-    }
-  } else {
-    rows = result.rows
-  }
-
-  if (!tableName) {
-    ctx.throw(400, "Could not find table name.")
-  }
-  let schema = datasource.entities[tableName].schema
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-
-  let headers = Object.keys(schema)
-
-  // @ts-ignore
-  const exporter = exporters[format]
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, exportRows))
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId

packages/server/src/api/controllers/row/index.ts

@@ -5,6 +5,9 @@ import { isExternalTable } from "../../../integrations/utils"
 import { Ctx } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
+import sdk from "../../../sdk"
+import * as exporters from "../view/exporters"
+import { apiFileReturn } from "../../../utilities/fileSystem"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -64,14 +67,26 @@ export const save = async (ctx: any) => {
 }
 export async function fetchView(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
+  const viewName = decodeURIComponent(ctx.params.viewName)
-    datasourceId: tableId,
+
-  })
+  const { calculation, group, field } = ctx.query
+
+  ctx.body = await quotas.addQuery(
+    () =>
+      sdk.rows.fetchView(tableId, viewName, {
+        calculation,
+        group,
+        field,
+      }),
+    {
+      datasourceId: tableId,
+    }
+  )
 }
 
 export async function fetch(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
     datasourceId: tableId,
   })
 }
@@ -119,8 +134,14 @@ export async function destroy(ctx: any) {
 
 export async function search(ctx: any) {
   const tableId = utils.getTableId(ctx)
+
+  const searchParams = {
+    ...ctx.request.body,
+    tableId,
+  }
+
   ctx.status = 200
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
     datasourceId: tableId,
   })
 }
@@ -150,7 +171,33 @@ export async function fetchEnrichedRow(ctx: any) {
 
 export const exportRows = async (ctx: any) => {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
+
-    datasourceId: tableId,
+  const format = ctx.query.format
-  })
+
+  const { rows, columns, query } = ctx.request.body
+  if (typeof format !== "string" || !exporters.isFormat(format)) {
+    ctx.throw(
+      400,
+      `Format ${format} not valid. Valid values: ${Object.values(
+        exporters.Format
+      )}`
+    )
+  }
+
+  ctx.body = await quotas.addQuery(
+    async () => {
+      const { fileName, content } = await sdk.rows.exportRows({
+        tableId,
+        format,
+        rowIds: rows,
+        columns,
+        query,
+      })
+      ctx.attachment(fileName)
+      return apiFileReturn(content)
+    },
+    {
+      datasourceId: tableId,
+    }
+  )
 }

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,7 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
-  getRowParams,
   getTableIDFromRowID,
-  DocumentType,
   InternalTables,
 } from "../../../db/utils"
 import * as userController from "../user"
@@ -14,78 +12,11 @@ import {
 } from "../../../utilities/rowProcessor"
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
-import { fullSearch, paginatedSearch } from "./internalSearch"
-import { getGlobalUsersFromMetadata } from "../../../utilities/global"
-import * as inMemoryViews from "../../../db/inMemoryView"
-import env from "../../../environment"
-import {
-  migrateToInMemoryView,
-  migrateToDesignView,
-  getFromDesignDoc,
-  getFromMemoryDoc,
-} from "../view/utils"
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format } from "../view/exporters"
+import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
-import { apiFileReturn } from "../../../utilities/fileSystem"
+import sdk from "../../../sdk"
-import {
-  UserCtx,
-  Database,
-  LinkDocumentValue,
-  Row,
-  Table,
-} from "@budibase/types"
-
-import { cleanExportRows } from "./utils"
-
-const CALCULATION_TYPES = {
-  SUM: "sum",
-  COUNT: "count",
-  STATS: "stats",
-}
-
-async function getView(db: Database, viewName: string) {
-  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
-  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
-  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
-  let viewInfo,
-    migrate = false
-  try {
-    viewInfo = await mainGetter(db, viewName)
-  } catch (err: any) {
-    // check if it can be retrieved from design doc (needs migrated)
-    if (err.status !== 404) {
-      viewInfo = null
-    } else {
-      viewInfo = await secondaryGetter(db, viewName)
-      migrate = !!viewInfo
-    }
-  }
-  if (migrate) {
-    await migration(db, viewName)
-  }
-  if (!viewInfo) {
-    throw "View does not exist."
-  }
-  return viewInfo
-}
-
-async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows as Row[]
-}
 
 export async function patch(ctx: UserCtx) {
   const db = context.getAppDB()
@@ -94,7 +25,7 @@ export async function patch(ctx: UserCtx) {
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
   try {
-    let dbTable = await db.get(tableId)
+    let dbTable = await sdk.tables.getTable(tableId)
     oldRow = await outputProcessing(
       dbTable,
       await utils.findRow(ctx, tableId, inputs._id)
@@ -110,7 +41,7 @@ export async function patch(ctx: UserCtx) {
       throw "Row does not exist"
     }
   }
-  let dbTable = await db.get(tableId)
+  let dbTable = await sdk.tables.getTable(tableId)
   // need to build up full patch fields before coerce
   let combinedRow: any = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
@@ -165,7 +96,7 @@ export async function save(ctx: UserCtx) {
   }
 
   // this returns the table and row incase they have been updated
-  const dbTable = await db.get(inputs.tableId)
+  const dbTable = await sdk.tables.getTable(inputs.tableId)
 
   // need to copy the table so it can be differenced on way out
   const tableClone = cloneDeep(dbTable)
@@ -195,85 +126,9 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: UserCtx) {
-  const viewName = decodeURIComponent(ctx.params.viewName)
-
-  // if this is a table view being looked for just transfer to that
-  if (viewName.startsWith(DocumentType.TABLE)) {
-    ctx.params.tableId = viewName
-    return fetch(ctx)
-  }
-
-  const db = context.getAppDB()
-  const { calculation, group, field } = ctx.query
-  const viewInfo = await getView(db, viewName)
-  let response
-  if (env.SELF_HOSTED) {
-    response = await db.query(`database/${viewName}`, {
-      include_docs: !calculation,
-      group: !!group,
-    })
-  } else {
-    const tableId = viewInfo.meta.tableId
-    const data = await getRawTableData(ctx, db, tableId)
-    response = await inMemoryViews.runView(
-      viewInfo,
-      calculation as string,
-      !!group,
-      data
-    )
-  }
-
-  let rows
-  if (!calculation) {
-    response.rows = response.rows.map(row => row.doc)
-    let table
-    try {
-      table = await db.get(viewInfo.meta.tableId)
-    } catch (err) {
-      /* istanbul ignore next */
-      table = {
-        schema: {},
-      }
-    }
-    rows = await outputProcessing(table, response.rows)
-  }
-
-  if (calculation === CALCULATION_TYPES.STATS) {
-    response.rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      ...row.value,
-      avg: row.value.sum / row.value.count,
-    }))
-    rows = response.rows
-  }
-
-  if (
-    calculation === CALCULATION_TYPES.COUNT ||
-    calculation === CALCULATION_TYPES.SUM
-  ) {
-    rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      value: row.value,
-    }))
-  }
-  return rows
-}
-
-export async function fetch(ctx: UserCtx) {
-  const db = context.getAppDB()
-
-  const tableId = ctx.params.tableId
-  let table = await db.get(tableId)
-  let rows = await getRawTableData(ctx, db, tableId)
-  return outputProcessing(table, rows)
-}
-
 export async function find(ctx: UserCtx) {
   const db = dbCore.getDB(ctx.appId)
-  const table = await db.get(ctx.params.tableId)
+  const table = await sdk.tables.getTable(ctx.params.tableId)
   let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
   row = await outputProcessing(table, row)
   return row
@@ -282,13 +137,13 @@ export async function find(ctx: UserCtx) {
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const { _id } = ctx.request.body
-  let row = await db.get(_id)
+  let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
 
   if (row.tableId !== ctx.params.tableId) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
-  const table = await db.get(row.tableId)
+  const table = await sdk.tables.getTable(row.tableId)
   // update the row to include full relationships before deleting them
   row = await outputProcessing(table, row, { squash: false })
   // now remove the relationships
@@ -318,7 +173,7 @@ export async function destroy(ctx: UserCtx) {
 export async function bulkDestroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
-  const table = await db.get(tableId)
+  const table = await sdk.tables.getTable(tableId)
   let { rows } = ctx.request.body
 
   // before carrying out any updates, make sure the rows are ready to be returned
@@ -354,108 +209,13 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: UserCtx) {
-  // Fetch the whole table when running in cypress, as search doesn't work
-  if (!env.COUCH_DB_URL && env.isCypress()) {
-    return { rows: await fetch(ctx) }
-  }
-
-  const { tableId } = ctx.params
-  const db = context.getAppDB()
-  const { paginate, query, ...params } = ctx.request.body
-  params.version = ctx.version
-  params.tableId = tableId
-
-  let table
-  if (params.sort && !params.sortType) {
-    table = await db.get(tableId)
-    const schema = table.schema
-    const sortField = schema[params.sort]
-    params.sortType = sortField.type == "number" ? "number" : "string"
-  }
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(query, params)
-  } else {
-    response = await fullSearch(query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    // enrich with global users if from users table
-    if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows)
-    }
-    table = table || (await db.get(tableId))
-    response.rows = await outputProcessing(table, response.rows)
-  }
-
-  return response
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const db = context.getAppDB()
-  const table = await db.get(ctx.params.tableId)
-  const rowIds = ctx.request.body.rows
-  let format = ctx.query.format
-  if (typeof format !== "string") {
-    ctx.throw(400, "Format parameter is not valid")
-  }
-  const { columns, query } = ctx.request.body
-
-  let result
-  if (rowIds) {
-    let response = (
-      await db.allDocs({
-        include_docs: true,
-        keys: rowIds,
-      })
-    ).rows.map(row => row.doc)
-
-    result = await outputProcessing(table, response)
-  } else if (query) {
-    let searchResponse = await search(ctx)
-    result = searchResponse.rows
-  }
-
-  let rows: Row[] = []
-  let schema = table.schema
-
-  // Filter data to only specified columns if required
-  if (columns && columns.length) {
-    for (let i = 0; i < result.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result[i][column]
-      }
-    }
-  } else {
-    rows = result
-  }
-
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-  if (format === Format.CSV) {
-    ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
-  } else if (format === Format.JSON) {
-    ctx.attachment("export.json")
-    return apiFileReturn(json(exportRows))
-  } else if (format === Format.JSON_WITH_SCHEMA) {
-    ctx.attachment("export.json")
-    return apiFileReturn(jsonWithSchema(schema, exportRows))
-  } else {
-    throw "Format not recognised"
-  }
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId
   // need table to work out where links go in row
   let [table, row] = await Promise.all([
-    db.get(tableId),
+    sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
   ])
   // get the link docs

packages/server/src/api/controllers/row/staticFormula.ts

@@ -8,8 +8,9 @@ import { FieldTypes, FormulaTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { Table, Row } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
-const { isEqual } = require("lodash")
+import sdk from "../../../sdk"
-const { cloneDeep } = require("lodash/fp")
+import { isEqual } from "lodash"
+import { cloneDeep } from "lodash/fp"
 
 /**
  * This function runs through a list of enriched rows, looks at the rows which
@@ -148,7 +149,7 @@ export async function finaliseRow(
       await db.put(table)
     } catch (err: any) {
       if (err.status === 409) {
-        const updatedTable = await db.get(table._id)
+        const updatedTable = await sdk.tables.getTable(table._id)
         let response = processAutoColumn(null, updatedTable, row, {
           reprocessing: true,
         })

packages/server/src/api/controllers/row/utils.ts

@@ -1,14 +1,19 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
-import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
-import { makeExternalQuery } from "../../../integrations/base/query"
+import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldType, Row, Table, UserCtx } from "@budibase/types"
+import { FieldTypes } from "../../../constants"
-import { Format } from "../view/exporters"
 import sdk from "../../../sdk"
 
-const validateJs = require("validate.js")
+import validateJs from "validate.js"
-const { cloneDeep } = require("lodash/fp")
+import { cloneDeep } from "lodash/fp"
+
+function isForeignKey(key: string, table: Table) {
+  const relationships = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  return relationships.some(relationship => relationship.foreignKey === key)
+}
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
-  )
-  return relationships.some(relationship => relationship.foreignKey === key)
-}
-
-export async function getDatasourceAndQuery(json: any) {
-  const datasourceId = json.endpoint.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId)
-  return makeExternalQuery(datasource, json)
-}
-
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
@@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   return row
 }
 
+export function getTableId(ctx: Ctx) {
+  if (ctx.request.body && ctx.request.body.tableId) {
+    return ctx.request.body.tableId
+  }
+  if (ctx.params && ctx.params.tableId) {
+    return ctx.params.tableId
+  }
+  if (ctx.params && ctx.params.viewName) {
+    return ctx.params.viewName
+  }
+}
+
 export async function validate({
   tableId,
   row,
@@ -81,8 +85,8 @@ export async function validate({
       continue
     }
     // special case for options, need to always allow unselected (empty)
-    if (type === FieldTypes.OPTIONS && constraints.inclusion) {
+    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
-      constraints.inclusion.push(null, "")
+      constraints.inclusion.push(null as any, "")
     }
     let res
 
@@ -94,13 +98,13 @@ export async function validate({
         }
         row[fieldName].map((val: any) => {
           if (
-            !constraints.inclusion.includes(val) &&
+            !constraints?.inclusion?.includes(val) &&
-            constraints.inclusion.length !== 0
+            constraints?.inclusion?.length !== 0
           ) {
             errors[fieldName] = "Field not in list"
           }
         })
-      } else if (constraints.presence && row[fieldName].length === 0) {
+      } else if (constraints?.presence && row[fieldName].length === 0) {
         // non required MultiSelect creates an empty array, which should not throw errors
         errors[fieldName] = [`${fieldName} is required`]
       }
@@ -128,52 +132,3 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
-
-export function cleanExportRows(
-  rows: any[],
-  schema: any,
-  format: string,
-  columns: string[]
-) {
-  let cleanRows = [...rows]
-
-  const relationships = Object.entries(schema)
-    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-
-  relationships.forEach(column => {
-    cleanRows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  if (format === Format.CSV) {
-    // Intended to append empty values in export
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      if (columns?.length && columns.indexOf(key) > 0) {
-        continue
-      }
-      for (let row of cleanRows) {
-        if (row[key] == null) {
-          row[key] = undefined
-        }
-      }
-    }
-  }
-
-  return cleanRows
-}
-
-export function getTableId(ctx: any) {
-  if (ctx.request.body && ctx.request.body.tableId) {
-    return ctx.request.body.tableId
-  }
-  if (ctx.params && ctx.params.tableId) {
-    return ctx.params.tableId
-  }
-  if (ctx.params && ctx.params.viewName) {
-    return ctx.params.viewName
-  }
-}

packages/server/src/api/controllers/screen.ts

@@ -7,7 +7,7 @@ import {
   roles,
 } from "@budibase/backend-core"
 import { updateAppPackage } from "./application"
-import { Plugin, ScreenProps, BBContext } from "@budibase/types"
+import { Plugin, ScreenProps, BBContext, Screen } from "@budibase/types"
 import { builderSocket } from "../../websockets"
 
 export async function fetch(ctx: BBContext) {
@@ -64,7 +64,7 @@ export async function save(ctx: BBContext) {
       })
 
     // Update the app metadata
-    const application = await db.get(DocumentType.APP_METADATA)
+    const application = await db.get<any>(DocumentType.APP_METADATA)
     let usedPlugins = application.usedPlugins || []
 
     requiredPlugins.forEach((plugin: Plugin) => {
@@ -104,7 +104,7 @@ export async function save(ctx: BBContext) {
 export async function destroy(ctx: BBContext) {
   const db = context.getAppDB()
   const id = ctx.params.screenId
-  const screen = await db.get(id)
+  const screen = await db.get<Screen>(id)
 
   await db.remove(id, ctx.params.screenRev)
 

packages/server/src/api/controllers/static/index.ts

@@ -1,7 +1,7 @@
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
-const uuid = require("uuid")
+import uuid from "uuid"
 import { ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {
@@ -16,6 +16,7 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
+import { App } from "@budibase/types"
 
 const send = require("koa-send")
 
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
   let db
   try {
     db = context.getAppDB({ skip_setup: true })
-    const appInfo = await db.get(DocumentType.APP_METADATA)
+    const appInfo = await db.get<any>(DocumentType.APP_METADATA)
     let appId = context.getAppId()
 
     if (!env.isJest()) {
@@ -177,7 +178,7 @@ export const serveApp = async function (ctx: any) {
 
 export const serveBuilderPreview = async function (ctx: any) {
   const db = context.getAppDB({ skip_setup: true })
-  const appInfo = await db.get(DocumentType.APP_METADATA)
+  const appInfo = await db.get<App>(DocumentType.APP_METADATA)
 
   if (!env.isJest()) {
     let appId = context.getAppId()

packages/server/src/api/controllers/table/external.ts

@@ -323,7 +323,7 @@ export async function save(ctx: UserCtx) {
 
   // Since tables are stored inside datasources, we need to notify clients
   // that the datasource definition changed
-  const updatedDatasource = await db.get(datasource._id)
+  const updatedDatasource = await sdk.datasources.get(datasource._id!)
   builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
 
   return tableToSave
@@ -354,7 +354,7 @@ export async function destroy(ctx: UserCtx) {
 
   // Since tables are stored inside datasources, we need to notify clients
   // that the datasource definition changed
-  const updatedDatasource = await db.get(datasource._id)
+  const updatedDatasource = await sdk.datasources.get(datasource._id!)
   builderSocket?.emitDatasourceUpdate(ctx, updatedDatasource)
 
   return tableToDelete

packages/server/src/api/controllers/table/internal.ts

@@ -15,7 +15,7 @@ import { isEqual } from "lodash"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
 
-function checkAutoColumns(table: Table, oldTable: Table) {
+function checkAutoColumns(table: Table, oldTable?: Table) {
   if (!table.schema) {
     return table
   }
@@ -46,7 +46,7 @@ export async function save(ctx: any) {
   // if the table obj had an _id then it will have been retrieved
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await db.get(ctx.request.body._id)
+    oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
 
   // check all types are correct
@@ -70,8 +70,8 @@ export async function save(ctx: any) {
   if (oldTable && oldTable.schema) {
     for (let propKey of Object.keys(tableToSave.schema)) {
       let oldColumn = oldTable.schema[propKey]
-      if (oldColumn && oldColumn.type === "internal") {
+      if (oldColumn && oldColumn.type === FieldTypes.INTERNAL) {
-        oldColumn.type = "auto"
+        oldColumn.type = FieldTypes.AUTO
       }
     }
   }
@@ -138,7 +138,7 @@ export async function save(ctx: any) {
 
 export async function destroy(ctx: any) {
   const db = context.getAppDB()
-  const tableToDelete = await db.get(ctx.params.tableId)
+  const tableToDelete = await sdk.tables.getTable(ctx.params.tableId)
 
   // Delete all rows for that table
   const rowsData = await db.allDocs(
@@ -160,7 +160,7 @@ export async function destroy(ctx: any) {
   })
 
   // don't remove the table itself until very end
-  await db.remove(tableToDelete._id, tableToDelete._rev)
+  await db.remove(tableToDelete._id!, tableToDelete._rev)
 
   // remove table search index
   if (!env.isTest() || env.COUCH_DB_URL) {
@@ -184,7 +184,6 @@ export async function destroy(ctx: any) {
 }
 
 export async function bulkImport(ctx: any) {
-  const db = context.getAppDB()
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
   await handleDataImport(ctx.user, table, rows, identifierFields)

packages/server/src/api/controllers/table/utils.ts

@@ -20,16 +20,10 @@ import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
 import { events, context } from "@budibase/backend-core"
-import {
+import { ContextUser, Datasource, SourceName, Table } from "@budibase/types"
-  ContextUser,
-  Database,
-  Datasource,
-  SourceName,
-  Table,
-} from "@budibase/types"
 
 export async function clearColumns(table: any, columnNames: any) {
-  const db: Database = context.getAppDB()
+  const db = context.getAppDB()
   const rows = await db.allDocs(
     getRowParams(table._id, null, {
       include_docs: true,

packages/server/src/api/controllers/tests/row.spec.ts

@@ -1,120 +0,0 @@
-import { exportRows } from "../row/external"
-import sdk from "../../../sdk"
-import { ExternalRequest } from "../row/ExternalRequest"
-
-// @ts-ignore
-sdk.datasources = {
-  get: jest.fn(),
-}
-
-jest.mock("../row/ExternalRequest")
-jest.mock("../view/exporters", () => ({
-  csv: jest.fn(),
-  Format: {
-    CSV: "csv",
-  },
-}))
-jest.mock("../../../utilities/fileSystem")
-
-function getUserCtx() {
-  return {
-    params: {
-      tableId: "datasource__tablename",
-    },
-    query: {
-      format: "csv",
-    },
-    request: {
-      body: {},
-    },
-    throw: jest.fn(() => {
-      throw "Err"
-    }),
-    attachment: jest.fn(),
-  }
-}
-
-describe("external row controller", () => {
-  describe("exportRows", () => {
-    beforeAll(() => {
-      //@ts-ignore
-      jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
-    })
-
-    afterEach(() => {
-      jest.clearAllMocks()
-    })
-
-    it("should throw a 400 if no datasource entities are present", async () => {
-      let userCtx = getUserCtx()
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Datasource has not been configured for plus API."
-        )
-      }
-    })
-
-    it("should handle single quotes from a row ID", async () => {
-      //@ts-ignore
-      sdk.datasources.get.mockImplementation(() => ({
-        entities: {
-          tablename: {
-            schema: {},
-          },
-        },
-      }))
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["['d001']"],
-      }
-
-      //@ts-ignore
-      await exportRows(userCtx)
-
-      expect(userCtx.request.body).toEqual({
-        query: {
-          oneOf: {
-            _id: ["d001"],
-          },
-        },
-      })
-    })
-
-    it("should throw a 400 if any composite keys are present", async () => {
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["[123]", "['d001'%2C'10111']"],
-      }
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Export data does not support composite keys."
-        )
-      }
-    })
-
-    it("should throw a 400 if no table name was found", async () => {
-      let userCtx = getUserCtx()
-      userCtx.params.tableId = "datasource__"
-      userCtx.request.body = {
-        rows: ["[123]"],
-      }
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Could not find table name."
-        )
-      }
-    })
-  })
-})

packages/server/src/api/controllers/user.ts

@@ -3,25 +3,11 @@ import { InternalTables } from "../../db/utils"
 import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function fetchMetadata(ctx: UserCtx) {
+export async function fetchMetadata(ctx: Ctx) {
-  const global = await getGlobalUsers()
+  const users = await sdk.users.fetchMetadata()
-  const metadata = await sdk.users.rawUserMetadata()
-  const users = []
-  for (let user of global) {
-    // find the metadata that matches up to the global ID
-    const info = metadata.find(meta => meta._id.includes(user._id))
-    // remove these props, not for the correct DB
-    users.push({
-      ...user,
-      ...info,
-      tableId: InternalTables.USER_METADATA,
-      // make sure the ID is always a local ID, not a global one
-      _id: generateUserMetadataID(user._id),
-    })
-  }
   ctx.body = users
 }
 
@@ -50,8 +36,8 @@ export async function updateMetadata(ctx: UserCtx) {
 export async function destroyMetadata(ctx: UserCtx) {
   const db = context.getAppDB()
   try {
-    const dbUser = await db.get(ctx.params.id)
+    const dbUser = await sdk.users.get(ctx.params.id)
-    await db.remove(dbUser._id, dbUser._rev)
+    await db.remove(dbUser._id!, dbUser._rev)
   } catch (err) {
     // error just means the global user has no config in this app
   }
@@ -74,7 +60,7 @@ export async function setFlag(ctx: UserCtx) {
   const db = context.getAppDB()
   let doc
   try {
-    doc = await db.get(flagDocId)
+    doc = await db.get<any>(flagDocId)
   } catch (err) {
     doc = { _id: flagDocId }
   }

packages/server/src/api/controllers/view/index.ts

@@ -15,7 +15,6 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
-import { cleanExportRows } from "../row/utils"
 import { builderSocket } from "../../../websockets"
 
 const { cloneDeep, isEqual } = require("lodash")
@@ -28,7 +27,8 @@ export async function save(ctx: Ctx) {
   const db = context.getAppDB()
   const { originalName, ...viewToSave } = ctx.request.body
 
-  const existingTable = await db.get(ctx.request.body.tableId)
+  const existingTable = await sdk.tables.getTable(ctx.request.body.tableId)
+  existingTable.views ??= {}
   const table = cloneDeep(existingTable)
 
   const groupByField: any = Object.values(table.schema).find(
@@ -121,8 +121,8 @@ export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const viewName = decodeURIComponent(ctx.params.viewName)
   const view = await deleteView(viewName)
-  const table = await db.get(view.meta.tableId)
+  const table = await sdk.tables.getTable(view.meta.tableId)
-  delete table.views[viewName]
+  delete table.views![viewName]
   await db.put(table)
   await events.view.deleted(view)
 
@@ -163,13 +163,16 @@ export async function exportView(ctx: Ctx) {
   let rows = ctx.body as Row[]
 
   let schema: TableSchema = view && view.meta && view.meta.schema
-  const tableId = ctx.params.tableId || view.meta.tableId
+  const tableId =
+    ctx.params.tableId ||
+    view?.meta?.tableId ||
+    (viewName.startsWith(DocumentType.TABLE) && viewName)
   const table: Table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, [])
+  let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)

packages/server/src/api/controllers/view/utils.ts

@@ -13,7 +13,7 @@ import { Database } from "@budibase/types"
 export async function getView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get("_design/database")
+    const designDoc = await db.get<any>("_design/database")
     return designDoc.views[viewName]
   } else {
     // This is a table view, don't read the view from the DB
@@ -22,7 +22,7 @@ export async function getView(viewName: string) {
     }
 
     try {
-      const viewDoc = await db.get(generateMemoryViewID(viewName))
+      const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
       return viewDoc.view
     } catch (err: any) {
       // Return null when PouchDB doesn't found the view
@@ -39,7 +39,7 @@ export async function getViews() {
   const db = context.getAppDB()
   const response = []
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get("_design/database")
+    const designDoc = await db.get<any>("_design/database")
     for (let name of Object.keys(designDoc.views)) {
       // Only return custom views, not built ins
       const viewNames = Object.values(ViewName) as string[]
@@ -76,7 +76,7 @@ export async function saveView(
 ) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get("_design/database")
+    const designDoc = await db.get<any>("_design/database")
     designDoc.views = {
       ...designDoc.views,
       [viewName]: viewTemplate,
@@ -96,9 +96,9 @@ export async function saveView(
       tableId: viewTemplate.meta.tableId,
     }
     try {
-      const old = await db.get(id)
+      const old = await db.get<any>(id)
       if (originalId) {
-        const originalDoc = await db.get(originalId)
+        const originalDoc = await db.get<any>(originalId)
         await db.remove(originalDoc._id, originalDoc._rev)
       }
       if (old && old._rev) {
@@ -114,14 +114,14 @@ export async function saveView(
 export async function deleteView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get("_design/database")
+    const designDoc = await db.get<any>("_design/database")
     const view = designDoc.views[viewName]
     delete designDoc.views[viewName]
     await db.put(designDoc)
     return view
   } else {
     const id = generateMemoryViewID(viewName)
-    const viewDoc = await db.get(id)
+    const viewDoc = await db.get<any>(id)
     await db.remove(viewDoc._id, viewDoc._rev)
     return viewDoc.view
   }
@@ -129,7 +129,7 @@ export async function deleteView(viewName: string) {
 
 export async function migrateToInMemoryView(db: Database, viewName: string) {
   // delete the view initially
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   // run the view back through the view builder to update it
   const view = viewBuilder(designDoc.views[viewName].meta)
   delete designDoc.views[viewName]
@@ -138,15 +138,15 @@ export async function migrateToInMemoryView(db: Database, viewName: string) {
 }
 
 export async function migrateToDesignView(db: Database, viewName: string) {
-  let view = await db.get(generateMemoryViewID(viewName))
+  let view = await db.get<any>(generateMemoryViewID(viewName))
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   designDoc.views[viewName] = viewBuilder(view.view.meta)
   await db.put(designDoc)
   await db.remove(view._id, view._rev)
 }
 
 export async function getFromDesignDoc(db: Database, viewName: string) {
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   let view = designDoc.views[viewName]
   if (view == null) {
     throw { status: 404, message: "Unable to get view" }
@@ -155,7 +155,7 @@ export async function getFromDesignDoc(db: Database, viewName: string) {
 }
 
 export async function getFromMemoryDoc(db: Database, viewName: string) {
-  let view = await db.get(generateMemoryViewID(viewName))
+  let view = await db.get<any>(generateMemoryViewID(viewName))
   if (view) {
     view = view.view
   } else {

packages/server/src/api/controllers/webhook.ts

@@ -77,7 +77,7 @@ export async function trigger(ctx: BBContext) {
       if (webhook.bodySchema) {
         validate(ctx.request.body, webhook.bodySchema)
       }
-      const target = await db.get(webhook.action.target)
+      const target = await db.get<Automation>(webhook.action.target)
       if (webhook.action.type === WebhookActionType.AUTOMATION) {
         // trigger with both the pure request and then expand it
         // incase the user has produced a schema to bind to

packages/server/src/api/routes/tests/internalSearch.spec.js

@@ -1,197 +0,0 @@
-const fetch = require("node-fetch")
-fetch.mockSearch()
-const search = require("../../controllers/row/internalSearch")
-// this will be mocked out for _search endpoint
-const PARAMS = {
-  tableId: "ta_12345679abcdef",
-  version: "1",
-  bookmark: null,
-  sort: null,
-  sortOrder: "ascending",
-  sortType: "string",
-}
-
-function checkLucene(resp, expected, params = PARAMS) {
-  const query = resp.rows[0].query
-  const json = JSON.parse(query)
-  if (PARAMS.sort) {
-    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-  }
-  if (PARAMS.bookmark) {
-    expect(json.bookmark).toBe(PARAMS.bookmark)
-  }
-  expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
-  expect(json.limit).toBe(params.limit || 50)
-}
-
-describe("internal search", () => {
-  it("default query", async () => {
-    const response = await search.paginatedSearch({
-    }, PARAMS)
-    checkLucene(response, `*:*`)
-  })
-
-  it("test equal query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:"1"`)
-  })
-
-  it("test notEqual query", async () => {
-    const response = await search.paginatedSearch({
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND !column:"1"`)
-  })
-
-  it("test OR query", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(column:"2" OR !column:"1")`)
-  })
-
-  it("test AND query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
-  })
-
-  it("test pagination query", async () => {
-    const updatedParams = {
-      ...PARAMS,
-      limit: 100,
-      bookmark: "awd",
-      sort: "column",
-    }
-    const response = await search.paginatedSearch({
-      string: {
-        "column": "2",
-      },
-    }, updatedParams)
-    checkLucene(response, `*:* AND column:2*`, updatedParams)
-  })
-
-  it("test range query", async () => {
-    const response = await search.paginatedSearch({
-      range: {
-        "column": { low: 1, high: 2 },
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
-  })
-
-  it("test empty query", async () => {
-    const response = await search.paginatedSearch({
-      empty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
-  })
-
-  it("test notEmpty query", async () => {
-    const response = await search.paginatedSearch({
-      notEmpty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
-  })
-
-  it("test oneOf query", async () => {
-    const response = await search.paginatedSearch({
-      oneOf: {
-        "column": ["a", "b"],
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
-  })
-
-  it("test contains query", async () => {
-    const response = await search.paginatedSearch({
-      contains: {
-        "column": "a",
-        "colArr": [1, 2, 3],
-      },
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
-  })
-
-  it("test multiple of same column", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "1:column": "a",
-        "2:column": "b",
-        "3:column": "c",
-      },
-    }, PARAMS)
-    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
-  })
-
-  it("check a weird case for lucene building", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "1:1:column": "a",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
-  })
-
-  it("test containsAny query", async () => {
-    const response = await search.paginatedSearch({
-      containsAny: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
-  })
-
-  it("test notContains query", async () => {
-    const response = await search.paginatedSearch({
-      notContains: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
-  })
-
-  it("test equal without version query", async () => {
-    PARAMS.version = null
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    
-    const query = response.rows[0].query
-    const json = JSON.parse(query)
-    if (PARAMS.sort) {
-      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-    }
-    if (PARAMS.bookmark) {
-      expect(json.bookmark).toBe(PARAMS.bookmark)
-    }
-    expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
-  })
-})

packages/server/src/api/routes/tests/internalSearch.spec.ts

@@ -0,0 +1,248 @@
+const nodeFetch = require("node-fetch")
+nodeFetch.mockSearch()
+import { SearchParams } from "@budibase/backend-core"
+import * as search from "../../../sdk/app/rows/search/internalSearch"
+import { Row } from "@budibase/types"
+
+// this will be mocked out for _search endpoint
+const PARAMS: SearchParams<Row> = {
+  tableId: "ta_12345679abcdef",
+  version: "1",
+  bookmark: undefined,
+  sort: undefined,
+  sortOrder: "ascending",
+  sortType: "string",
+}
+
+function checkLucene(resp: any, expected: any, params = PARAMS) {
+  const query = resp.rows[0].query
+  const json = JSON.parse(query)
+  if (PARAMS.sort) {
+    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+  }
+  if (PARAMS.bookmark) {
+    expect(json.bookmark).toBe(PARAMS.bookmark)
+  }
+  expect(json.include_docs).toBe(true)
+  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
+  expect(json.limit).toBe(params.limit || 50)
+}
+
+describe("internal search", () => {
+  it("default query", async () => {
+    const response = await search.paginatedSearch({}, PARAMS)
+    checkLucene(response, `*:*`)
+  })
+
+  it("test equal query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:"1"`)
+  })
+
+  it("test notEqual query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND !column:"1"`)
+  })
+
+  it("test OR query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"2" OR !column:"1")`)
+  })
+
+  it("test AND query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
+  })
+
+  it("test pagination query", async () => {
+    const updatedParams = {
+      ...PARAMS,
+      limit: 100,
+      bookmark: "awd",
+      sort: "column",
+    }
+    const response = await search.paginatedSearch(
+      {
+        string: {
+          column: "2",
+        },
+      },
+      updatedParams
+    )
+    checkLucene(response, `*:* AND column:2*`, updatedParams)
+  })
+
+  it("test range query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        range: {
+          column: { low: 1, high: 2 },
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
+  })
+
+  it("test empty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        empty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
+  })
+
+  it("test notEmpty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEmpty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
+  })
+
+  it("test oneOf query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        oneOf: {
+          column: ["a", "b"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
+  })
+
+  it("test contains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        contains: {
+          column: "a",
+          colArr: [1, 2, 3],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(
+      response,
+      `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
+      PARAMS
+    )
+  })
+
+  it("test multiple of same column", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          "1:column": "a",
+          "2:column": "b",
+          "3:column": "c",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
+  })
+
+  it("check a weird case for lucene building", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          "1:1:column": "a",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
+  })
+
+  it("test containsAny query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        containsAny: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
+  })
+
+  it("test notContains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notContains: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
+  })
+
+  it("test equal without version query", async () => {
+    PARAMS.version = undefined
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+
+    const query = response.rows[0].query
+    const json = JSON.parse(query)
+    if (PARAMS.sort) {
+      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+    }
+    if (PARAMS.bookmark) {
+      expect(json.bookmark).toBe(PARAMS.bookmark)
+    }
+    expect(json.include_docs).toBe(true)
+    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
+  })
+})

packages/server/src/api/routes/tests/row.spec.ts

@@ -1,27 +1,27 @@
-const tk = require( "timekeeper")
+import tk from "timekeeper"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
-
+import { outputProcessing } from "../../../utilities/rowProcessor"
-const { outputProcessing } = require("../../../utilities/rowProcessor")
+import * as setup from "./utilities"
-const setup = require("./utilities")
 const { basicRow } = setup.structures
-const { context, tenancy } = require("@budibase/backend-core")
+import { context, tenancy } from "@budibase/backend-core"
-const {
+import { quotas } from "@budibase/pro"
-  quotas,
+import {
-} = require("@budibase/pro")
-const {
   QuotaUsageType,
   StaticQuotaName,
   MonthlyQuotaName,
-} = require("@budibase/types")
+  Row,
-const { structures } = require("@budibase/backend-core/tests");
+  Table,
+  FieldType,
+} from "@budibase/types"
+import { structures } from "@budibase/backend-core/tests"
 
 describe("/rows", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let table
+  let table: Table
-  let row
+  let row: Row
 
   afterAll(setup.afterAll)
 
@@ -29,12 +29,12 @@ describe("/rows", () => {
     await config.init()
   })
 
-  beforeEach(async()=>{
+  beforeEach(async () => {
     table = await config.createTable()
-    row = basicRow(table._id)
+    row = basicRow(table._id!)
   })
 
-  const loadRow = async (id, tbl_Id, status = 200) =>
+  const loadRow = async (id: string, tbl_Id: string, status = 200) =>
     await request
       .get(`/api/${tbl_Id}/rows/${id}`)
       .set(config.defaultHeaders())
@@ -42,21 +42,28 @@ describe("/rows", () => {
       .expect(status)
 
   const getRowUsage = async () => {
-    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+    const { total } = await config.doInContext(null, () =>
+      quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
+    )
     return total
   }
 
   const getQueryUsage = async () => {
-    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+    const { total } = await config.doInContext(null, () =>
+      quotas.getCurrentUsageValues(
+        QuotaUsageType.MONTHLY,
+        MonthlyQuotaName.QUERIES
+      )
+    )
     return total
   }
 
-  const assertRowUsage = async expected => {
+  const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
   }
 
-  const assertQueryUsage = async expected => {
+  const assertQueryUsage = async (expected: number) => {
     const usage = await getQueryUsage()
     expect(usage).toBe(expected)
   }
@@ -70,9 +77,11 @@ describe("/rows", () => {
         .post(`/api/${row.tableId}/rows`)
         .send(row)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
+      expect((res as any).res.statusMessage).toEqual(
+        `${table.name} saved successfully`
+      )
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
       await assertRowUsage(rowUsage + 1)
@@ -86,12 +95,11 @@ describe("/rows", () => {
       const newTable = await config.createTable({
         name: "TestTableAuto",
         type: "table",
-        key: "name",
         schema: {
           ...table.schema,
           "Row ID": {
             name: "Row ID",
-            type: "number",
+            type: FieldType.NUMBER,
             subtype: "autoID",
             icon: "ri-magic-line",
             autocolumn: true,
@@ -104,28 +112,30 @@ describe("/rows", () => {
               },
             },
           },
-        }
+        },
       })
 
-      const ids = [1,2,3]
+      const ids = [1, 2, 3]
 
       // Performing several create row requests should increment the autoID fields accordingly
-      const createRow = async (id) => {
+      const createRow = async (id: number) => {
         const res = await request
           .post(`/api/${newTable._id}/rows`)
           .send({
-            name: "row_" + id
+            name: "row_" + id,
           })
           .set(config.defaultHeaders())
-          .expect('Content-Type', /json/)
+          .expect("Content-Type", /json/)
           .expect(200)
-        expect(res.res.statusMessage).toEqual(`${newTable.name} saved successfully`)
+        expect((res as any).res.statusMessage).toEqual(
+          `${newTable.name} saved successfully`
+        )
         expect(res.body.name).toEqual("row_" + id)
         expect(res.body._rev).toBeDefined()
         expect(res.body["Row ID"]).toEqual(id)
       }
 
-      for (let i=0; i<ids.length; i++ ){
+      for (let i = 0; i < ids.length; i++) {
         await createRow(ids[i])
       }
 
@@ -150,7 +160,7 @@ describe("/rows", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
@@ -196,8 +206,8 @@ describe("/rows", () => {
         .expect(200)
 
       expect(res.body.length).toBe(2)
-      expect(res.body.find(r => r.name === newRow.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find(r => r.name === row.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -215,92 +225,91 @@ describe("/rows", () => {
 
     it("row values are coerced", async () => {
       const str = {
-        type: "string",
+        type: FieldType.STRING,
+        name: "str",
         constraints: { type: "string", presence: false },
       }
       const attachment = {
-        type: "attachment",
+        type: FieldType.ATTACHMENT,
+        name: "attachment",
         constraints: { type: "array", presence: false },
       }
       const bool = {
-        type: "boolean",
+        type: FieldType.BOOLEAN,
+        name: "boolean",
         constraints: { type: "boolean", presence: false },
       }
       const number = {
-        type: "number",
+        type: FieldType.NUMBER,
+        name: "str",
         constraints: { type: "number", presence: false },
       }
       const datetime = {
-        type: "datetime",
+        type: FieldType.DATETIME,
+        name: "datetime",
         constraints: {
           type: "string",
           presence: false,
           datetime: { earliest: "", latest: "" },
-        }
+        },
       }
       const arrayField = {
-        type: "array",
+        type: FieldType.ARRAY,
         constraints: {
           type: "array",
           presence: false,
-          inclusion: [
+          inclusion: ["One", "Two", "Three"],
-            "One",
-            "Two",
-            "Three",
-          ]
         },
         name: "Sample Tags",
-        sortable: false
+        sortable: false,
       }
       const optsField = {
-        fieldName: "Sample Opts",
+          fieldName: "Sample Opts",
-        name: "Sample Opts",
+          name: "Sample Opts",
-        type: "options",
+          type: FieldType.OPTIONS,
-        constraints: {
+          constraints: {
-          type: "string",
+            type: "string",
-          presence: false,
+            presence: false,
-          inclusion: [ "Alpha", "Beta", "Gamma" ]
+            inclusion: ["Alpha", "Beta", "Gamma"],
+          },
         },
-      },
+        table = await config.createTable({
-
+          name: "TestTable2",
-      table = await config.createTable({
+          type: "table",
-        name: "TestTable2",
+          schema: {
-        type: "table",
+            name: str,
-        key: "name",
+            stringUndefined: str,
-        schema: {
+            stringNull: str,
-          name: str,
+            stringString: str,
-          stringUndefined: str,
+            numberEmptyString: number,
-          stringNull: str,
+            numberNull: number,
-          stringString: str,
+            numberUndefined: number,
-          numberEmptyString: number,
+            numberString: number,
-          numberNull: number,
+            numberNumber: number,
-          numberUndefined: number,
+            datetimeEmptyString: datetime,
-          numberString: number,
+            datetimeNull: datetime,
-          numberNumber: number,
+            datetimeUndefined: datetime,
-          datetimeEmptyString: datetime,
+            datetimeString: datetime,
-          datetimeNull: datetime,
+            datetimeDate: datetime,
-          datetimeUndefined: datetime,
+            boolNull: bool,
-          datetimeString: datetime,
+            boolEmpty: bool,
-          datetimeDate: datetime,
+            boolUndefined: bool,
-          boolNull: bool,
+            boolString: bool,
-          boolEmpty: bool,
+            boolBool: bool,
-          boolUndefined: bool,
+            attachmentNull: attachment,
-          boolString: bool,
+            attachmentUndefined: attachment,
-          boolBool: bool,
+            attachmentEmpty: attachment,
-          attachmentNull: attachment,
+            attachmentEmptyArrayStr: attachment,
-          attachmentUndefined: attachment,
+            arrayFieldEmptyArrayStr: arrayField,
-          attachmentEmpty: attachment,
+            arrayFieldArrayStrKnown: arrayField,
-          attachmentEmptyArrayStr: attachment,
+            arrayFieldNull: arrayField,
-          arrayFieldEmptyArrayStr: arrayField,
+            arrayFieldUndefined: arrayField,
-          arrayFieldArrayStrKnown: arrayField,
+            optsFieldEmptyStr: optsField,
-          arrayFieldNull: arrayField,
+            optsFieldUndefined: optsField,
-          arrayFieldUndefined: arrayField,
+            optsFieldNull: optsField,
-          optsFieldEmptyStr: optsField,
+            optsFieldStrKnown: optsField,
-          optsFieldUndefined: optsField,
+          },
-          optsFieldNull: optsField,
+        })
-          optsFieldStrKnown: optsField
-        },
-      })
 
       row = {
         name: "Test Row",
@@ -334,13 +343,13 @@ describe("/rows", () => {
         optsFieldEmptyStr: "",
         optsFieldUndefined: undefined,
         optsFieldNull: null,
-        optsFieldStrKnown: 'Alpha'
+        optsFieldStrKnown: "Alpha",
       }
 
-      const createdRow = await config.createRow(row);
+      const createdRow = await config.createRow(row)
-      const id = createdRow._id
+      const id = createdRow._id!
 
-      const saved = (await loadRow(id, table._id)).body
+      const saved = (await loadRow(id, table._id!)).body
 
       expect(saved.stringUndefined).toBe(undefined)
       expect(saved.stringNull).toBe("")
@@ -365,15 +374,15 @@ describe("/rows", () => {
       expect(saved.attachmentNull).toEqual([])
       expect(saved.attachmentUndefined).toBe(undefined)
       expect(saved.attachmentEmpty).toEqual([])
-      expect(saved.attachmentEmptyArrayStr).toEqual([])      
+      expect(saved.attachmentEmptyArrayStr).toEqual([])
       expect(saved.arrayFieldEmptyArrayStr).toEqual([])
       expect(saved.arrayFieldNull).toEqual([])
       expect(saved.arrayFieldUndefined).toEqual(undefined)
       expect(saved.optsFieldEmptyStr).toEqual(null)
       expect(saved.optsFieldUndefined).toEqual(undefined)
       expect(saved.optsFieldNull).toEqual(null)
-      expect(saved.arrayFieldArrayStrKnown).toEqual(['One'])
+      expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-      expect(saved.optsFieldStrKnown).toEqual('Alpha')
+      expect(saved.optsFieldStrKnown).toEqual("Alpha")
     })
   })
 
@@ -396,13 +405,13 @@ describe("/rows", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
       expect(res.body.description).toEqual(existing.description)
 
-      const savedRow = await loadRow(res.body._id, table._id)
+      const savedRow = await loadRow(res.body._id, table._id!)
 
       expect(savedRow.body.description).toEqual(existing.description)
       expect(savedRow.body.name).toEqual("Updated Name")
@@ -504,7 +513,7 @@ describe("/rows", () => {
         .expect(200)
 
       expect(res.body.length).toEqual(2)
-      await loadRow(row1._id, table._id, 404)
+      await loadRow(row1._id!, table._id!, 404)
       await assertRowUsage(rowUsage - 2)
       await assertQueryUsage(queryUsage + 1)
     })
@@ -562,7 +571,7 @@ describe("/rows", () => {
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
       const { table, firstRow, secondRow } = await tenancy.doInTenant(
-        setup.structures.TENANT_ID,
+        config.getTenantId(),
         async () => {
           const table = await config.createLinkedTable()
           const firstRow = await config.createRow({
@@ -624,7 +633,7 @@ describe("/rows", () => {
       await setup.switchToSelfHosted(async () => {
         context.doInAppContext(config.getAppId(), async () => {
           const enriched = await outputProcessing(table, [row])
-          expect(enriched[0].attachment[0].url).toBe(
+          expect((enriched as Row[])[0].attachment[0].url).toBe(
             `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
           )
         })

packages/server/src/automations/tests/queryRows.spec.ts

@@ -1,6 +1,6 @@
 // lucene searching not supported in test due to use of PouchDB
-let rows = []
+let rows: Row[] = []
-jest.mock("../../api/controllers/row/internalSearch", () => ({
+jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
   fullSearch: jest.fn(() => {
     return {
       rows,
@@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
   }),
   paginatedSearch: jest.fn(),
 }))
-const setup = require("./utilities")
+import { Row, Table } from "@budibase/types"
+import * as setup from "./utilities"
 
 const NAME = "Test"
 
 describe("Test a query step automation", () => {
-  let table
+  let table: Table
   let config = setup.getConfig()
 
   beforeAll(async () => {
@@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
       filters: {},
       "filters-def": [
         {
-          value: null
+          value: null,
-        }
+        },
       ],
       sortColumn: "name",
       sortOrder: "ascending",

packages/server/src/automations/utils.ts

@@ -8,7 +8,12 @@ import { db as dbCore, context } from "@budibase/backend-core"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { Automation, AutomationJob, WebhookActionType } from "@budibase/types"
+import {
+  Automation,
+  AutomationJob,
+  Webhook,
+  WebhookActionType,
+} from "@budibase/types"
 import sdk from "../sdk"
 
 const REBOOT_CRON = "@reboot"
@@ -204,15 +209,15 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     oldTrigger.webhookId
   ) {
     try {
-      let db = context.getAppDB()
+      const db = context.getAppDB()
       // need to get the webhook to get the rev
-      const webhook = await db.get(oldTrigger.webhookId)
+      const webhook = await db.get<Webhook>(oldTrigger.webhookId)
       // might be updating - reset the inputs to remove the URLs
       if (newTrigger) {
         delete newTrigger.webhookId
         newTrigger.inputs = {}
       }
-      await sdk.automations.webhook.destroy(webhook._id, webhook._rev)
+      await sdk.automations.webhook.destroy(webhook._id!, webhook._rev!)
     } catch (err) {
       // don't worry about not being able to delete, if it doesn't exist all good
     }

packages/server/src/db/linkedRows/LinkController.ts

@@ -182,7 +182,7 @@ class LinkController {
         })
 
         // if 1:N, ensure that this ID is not already attached to another record
-        const linkedTable = await this._db.get(field.tableId)
+        const linkedTable = await this._db.get<Table>(field.tableId)
         const linkedSchema = linkedTable.schema[field.fieldName!]
 
         // We need to map the global users to metadata in each app for relationships
@@ -311,7 +311,7 @@ class LinkController {
       })
     )
     // remove schema from other table
-    let linkedTable = await this._db.get(field.tableId)
+    let linkedTable = await this._db.get<Table>(field.tableId)
     if (field.fieldName) {
       delete linkedTable.schema[field.fieldName]
     }
@@ -337,7 +337,7 @@ class LinkController {
         // table for some reason
         let linkedTable
         try {
-          linkedTable = await this._db.get(field.tableId)
+          linkedTable = await this._db.get<Table>(field.tableId)
         } catch (err) {
           /* istanbul ignore next */
           continue
@@ -416,7 +416,7 @@ class LinkController {
       const field = schema[fieldName]
       try {
         if (field.type === FieldTypes.LINK && field.fieldName) {
-          const linkedTable = await this._db.get(field.tableId)
+          const linkedTable = await this._db.get<Table>(field.tableId)
           delete linkedTable.schema[field.fieldName]
           await this._db.put(linkedTable)
         }

packages/server/src/db/views/staticViews.ts

@@ -22,7 +22,7 @@ const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR
  */
 export async function createLinkView() {
   const db = context.getAppDB()
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   const view = {
     map: function (doc: LinkDocument) {
       // everything in this must remain constant as its going to Pouch, no external variables
@@ -58,7 +58,7 @@ export async function createLinkView() {
 
 export async function createRoutingView() {
   const db = context.getAppDB()
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   const view = {
     // if using variables in a map function need to inject them before use
     map: `function(doc) {
@@ -79,7 +79,7 @@ export async function createRoutingView() {
 
 async function searchIndex(indexName: string, fnString: string) {
   const db = context.getAppDB()
-  const designDoc = await db.get("_design/database")
+  const designDoc = await db.get<any>("_design/database")
   designDoc.indexes = {
     [indexName]: {
       index: fnString,

packages/server/src/integrations/tests/mysql.spec.ts

@@ -102,7 +102,7 @@ describe("MySQL Integration", () => {
       )
     })
 
-    it("parses strings matching a valid date format", async () => {
+    it.skip("parses strings matching a valid date format", async () => {
       const sql = "select * from users;"
       await config.integration.read({
         sql,

packages/server/src/middleware/builder.ts

@@ -10,7 +10,7 @@ import {
   setDebounce,
 } from "../utilities/redis"
 import { db as dbCore, cache } from "@budibase/backend-core"
-import { UserCtx, Database } from "@budibase/types"
+import { UserCtx, Database, App } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -51,7 +51,7 @@ async function updateAppUpdatedAt(ctx: UserCtx) {
   }
   await dbCore.doWithDB(appId, async (db: Database) => {
     try {
-      const metadata = await db.get(DocumentType.APP_METADATA)
+      const metadata = await db.get<any>(DocumentType.APP_METADATA)
       metadata.updatedAt = new Date().toISOString()
 
       metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)

packages/server/src/sdk/app/applications/sync.ts

@@ -37,7 +37,7 @@ async function syncUsersToApp(
 
       let metadata
       try {
-        metadata = await db.get(metadataId)
+        metadata = await db.get<any>(metadataId)
       } catch (err: any) {
         if (err.status !== 404) {
           throw err

packages/server/src/sdk/app/datasources/datasources.ts

@@ -62,7 +62,7 @@ export async function get(
   opts?: { enriched: boolean }
 ): Promise<Datasource> {
   const appDb = context.getAppDB()
-  const datasource = await appDb.get(datasourceId)
+  const datasource = await appDb.get<Datasource>(datasourceId)
   if (opts?.enriched) {
     return (await enrichDatasourceWithValues(datasource)).datasource
   } else {
@@ -72,7 +72,7 @@ export async function get(
 
 export async function getWithEnvVars(datasourceId: string) {
   const appDb = context.getAppDB()
-  const datasource = await appDb.get(datasourceId)
+  const datasource = await appDb.get<Datasource>(datasourceId)
   return enrichDatasourceWithValues(datasource)
 }
 

packages/server/src/sdk/app/rows/index.ts

@@ -1,7 +1,11 @@
 import * as attachments from "./attachments"
 import * as rows from "./rows"
+import * as search from "./search"
+import * as utils from "./utils"
 
 export default {
   ...attachments,
   ...rows,
+  ...search,
+  utils: utils,
 }

packages/server/src/sdk/app/rows/search.ts

@@ -0,0 +1,66 @@
+import { SearchFilters } from "@budibase/types"
+import { isExternalTable } from "../../../integrations/utils"
+import * as internal from "./search/internal"
+import * as external from "./search/external"
+import { Format } from "../../../api/controllers/view/exporters"
+
+export interface SearchParams {
+  tableId: string
+  paginate?: boolean
+  query: SearchFilters
+  bookmark?: string
+  limit?: number
+  sort?: string
+  sortOrder?: string
+  sortType?: string
+  version?: string
+  disableEscaping?: boolean
+}
+
+export interface ViewParams {
+  calculation: string
+  group: string
+  field: string
+}
+
+function pickApi(tableId: any) {
+  if (isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+export async function search(options: SearchParams) {
+  return pickApi(options.tableId).search(options)
+}
+
+export interface ExportRowsParams {
+  tableId: string
+  format: Format
+  rowIds?: string[]
+  columns?: string[]
+  query: SearchFilters
+}
+
+export interface ExportRowsResult {
+  fileName: string
+  content: string
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  return pickApi(options.tableId).exportRows(options)
+}
+
+export async function fetch(tableId: string) {
+  return pickApi(tableId).fetch(tableId)
+}
+
+export async function fetchView(
+  tableId: string,
+  viewName: string,
+  params: ViewParams
+) {
+  return pickApi(tableId).fetchView(viewName, params)
+}

packages/server/src/sdk/app/rows/search/external.ts

@@ -0,0 +1,172 @@
+import {
+  SortJson,
+  SortDirection,
+  Operation,
+  PaginationJson,
+  IncludeRelationship,
+  Row,
+  SearchFilters,
+} from "@budibase/types"
+import * as exporters from "../../../../api/controllers/view/exporters"
+import sdk from "../../../../sdk"
+import { handleRequest } from "../../../../api/controllers/row/external"
+import { breakExternalTableId } from "../../../../integrations/utils"
+import { cleanExportRows } from "../utils"
+import { utils } from "@budibase/shared-core"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { HTTPError } from "@budibase/backend-core"
+
+export async function search(options: SearchParams) {
+  const { tableId } = options
+  const { paginate, query, ...params } = options
+  const { limit } = params
+  let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
+  if (paginate && !bookmark) {
+    bookmark = 1
+  }
+  let paginateObj = {}
+
+  if (paginate) {
+    paginateObj = {
+      // add one so we can track if there is another page
+      limit: limit,
+      page: bookmark,
+    }
+  } else if (params && limit) {
+    paginateObj = {
+      limit: limit,
+    }
+  }
+  let sort: SortJson | undefined
+  if (params.sort) {
+    const direction =
+      params.sortOrder === "descending"
+        ? SortDirection.DESCENDING
+        : SortDirection.ASCENDING
+    sort = {
+      [params.sort]: { direction },
+    }
+  }
+  try {
+    const rows = (await handleRequest(Operation.READ, tableId, {
+      filters: query,
+      sort,
+      paginate: paginateObj as PaginationJson,
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
+    })) as Row[]
+    let hasNextPage = false
+    if (paginate && rows.length === limit) {
+      const nextRows = (await handleRequest(Operation.READ, tableId, {
+        filters: query,
+        sort,
+        paginate: {
+          limit: 1,
+          page: bookmark! * limit + 1,
+        },
+        includeSqlRelationships: IncludeRelationship.INCLUDE,
+      })) as Row[]
+      hasNextPage = nextRows.length > 0
+    }
+    // need wrapper object for bookmarks etc when paginating
+    return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
+  } catch (err: any) {
+    if (err.message && err.message.includes("does not exist")) {
+      throw new Error(
+        `Table updated externally, please re-fetch - ${err.message}`
+      )
+    } else {
+      throw err
+    }
+  }
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, columns, rowIds } = options
+  const { datasourceId, tableName } = breakExternalTableId(tableId)
+
+  let query: SearchFilters = {}
+  if (rowIds?.length) {
+    query = {
+      oneOf: {
+        _id: rowIds.map((row: string) => {
+          const ids = JSON.parse(
+            decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
+          )
+          if (ids.length > 1) {
+            throw new HTTPError(
+              "Export data does not support composite keys.",
+              400
+            )
+          }
+          return ids[0]
+        }),
+      },
+    }
+  }
+
+  const datasource = await sdk.datasources.get(datasourceId!)
+  if (!datasource || !datasource.entities) {
+    throw new HTTPError("Datasource has not been configured for plus API.", 400)
+  }
+
+  let result = await search({ tableId, query })
+  let rows: Row[] = []
+
+  // Filter data to only specified columns if required
+
+  if (columns && columns.length) {
+    for (let i = 0; i < result.rows.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result.rows[i][column]
+      }
+    }
+  } else {
+    rows = result.rows
+  }
+
+  if (!tableName) {
+    throw new HTTPError("Could not find table name.", 400)
+  }
+  const schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
+  let content: string
+  switch (format) {
+    case exporters.Format.CSV:
+      content = exporters.csv(headers, exportRows)
+      break
+    case exporters.Format.JSON:
+      content = exporters.json(exportRows)
+      break
+    case exporters.Format.JSON_WITH_SCHEMA:
+      content = exporters.jsonWithSchema(schema, exportRows)
+      break
+    default:
+      throw utils.unreachable(format)
+  }
+
+  const fileName = `export.${format}`
+  return {
+    fileName,
+    content,
+  }
+}
+
+export async function fetch(tableId: string) {
+  return handleRequest(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
+}
+
+export async function fetchView(viewName: string) {
+  // there are no views in external datasources, shouldn't ever be called
+  // for now just fetch
+  const split = viewName.split("all_")
+  const tableId = split[1] ? split[1] : split[0]
+  return fetch(tableId)
+}

packages/server/src/sdk/app/rows/search/internal.ts

@@ -0,0 +1,261 @@
+import {
+  context,
+  SearchParams as InternalSearchParams,
+} from "@budibase/backend-core"
+import env from "../../../../environment"
+import { fullSearch, paginatedSearch } from "./internalSearch"
+import {
+  InternalTables,
+  getRowParams,
+  DocumentType,
+} from "../../../../db/utils"
+import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
+import { Database, Row, Table } from "@budibase/types"
+import { cleanExportRows } from "../utils"
+import {
+  Format,
+  csv,
+  json,
+  jsonWithSchema,
+} from "../../../../api/controllers/view/exporters"
+import * as inMemoryViews from "../../../../db/inMemoryView"
+import {
+  migrateToInMemoryView,
+  migrateToDesignView,
+  getFromDesignDoc,
+  getFromMemoryDoc,
+} from "../../../../api/controllers/view/utils"
+import sdk from "../../../../sdk"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+
+export async function search(options: SearchParams) {
+  const { tableId } = options
+
+  // Fetch the whole table when running in cypress, as search doesn't work
+  if (!env.COUCH_DB_URL && env.isCypress()) {
+    return { rows: await fetch(tableId) }
+  }
+
+  const { paginate, query } = options
+
+  const params: InternalSearchParams<any> = {
+    tableId: options.tableId,
+    sort: options.sort,
+    sortOrder: options.sortOrder,
+    sortType: options.sortType,
+    limit: options.limit,
+    bookmark: options.bookmark,
+    version: options.version,
+    disableEscaping: options.disableEscaping,
+  }
+
+  let table
+  if (params.sort && !params.sortType) {
+    table = await sdk.tables.getTable(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type === "number" ? "number" : "string"
+  }
+
+  let response
+  if (paginate) {
+    response = await paginatedSearch(query, params)
+  } else {
+    response = await fullSearch(query, params)
+  }
+
+  // Enrich search results with relationships
+  if (response.rows && response.rows.length) {
+    // enrich with global users if from users table
+    if (tableId === InternalTables.USER_METADATA) {
+      response.rows = await getGlobalUsersFromMetadata(response.rows)
+    }
+    table = table || (await sdk.tables.getTable(tableId))
+    response.rows = await outputProcessing(table, response.rows)
+  }
+
+  return response
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, rowIds, columns, query } = options
+  const db = context.getAppDB()
+  const table = await sdk.tables.getTable(tableId)
+
+  let result
+  if (rowIds) {
+    let response = (
+      await db.allDocs({
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await search({ tableId, query })
+    result = searchResponse.rows
+  }
+
+  let rows: Row[] = []
+  let schema = table.schema
+
+  // Filter data to only specified columns if required
+  if (columns && columns.length) {
+    for (let i = 0; i < result.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result[i][column]
+      }
+    }
+  } else {
+    rows = result
+  }
+
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+  if (format === Format.CSV) {
+    return {
+      fileName: "export.csv",
+      content: csv(Object.keys(rows[0]), exportRows),
+    }
+  } else if (format === Format.JSON) {
+    return {
+      fileName: "export.json",
+      content: json(exportRows),
+    }
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    return {
+      fileName: "export.json",
+      content: jsonWithSchema(schema, exportRows),
+    }
+  } else {
+    throw "Format not recognised"
+  }
+}
+
+export async function fetch(tableId: string) {
+  const db = context.getAppDB()
+
+  let table = await sdk.tables.getTable(tableId)
+  let rows = await getRawTableData(db, tableId)
+  const result = await outputProcessing(table, rows)
+  return result
+}
+
+async function getRawTableData(db: Database, tableId: string) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    rows = await sdk.users.fetchMetadata()
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows as Row[]
+}
+
+export async function fetchView(
+  viewName: string,
+  options: { calculation: string; group: string; field: string }
+) {
+  // if this is a table view being looked for just transfer to that
+  if (viewName.startsWith(DocumentType.TABLE)) {
+    return fetch(viewName)
+  }
+
+  const db = context.getAppDB()
+  const { calculation, group, field } = options
+  const viewInfo = await getView(db, viewName)
+  let response
+  if (env.SELF_HOSTED) {
+    response = await db.query(`database/${viewName}`, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+  } else {
+    const tableId = viewInfo.meta.tableId
+    const data = await getRawTableData(db, tableId)
+    response = await inMemoryViews.runView(
+      viewInfo,
+      calculation as string,
+      !!group,
+      data
+    )
+  }
+
+  let rows
+  if (!calculation) {
+    response.rows = response.rows.map(row => row.doc)
+    let table: Table
+    try {
+      table = await sdk.tables.getTable(viewInfo.meta.tableId)
+    } catch (err) {
+      /* istanbul ignore next */
+      table = {
+        name: "",
+        schema: {},
+      }
+    }
+    rows = await outputProcessing(table, response.rows)
+  }
+
+  if (calculation === CALCULATION_TYPES.STATS) {
+    response.rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      ...row.value,
+      avg: row.value.sum / row.value.count,
+    }))
+    rows = response.rows
+  }
+
+  if (
+    calculation === CALCULATION_TYPES.COUNT ||
+    calculation === CALCULATION_TYPES.SUM
+  ) {
+    rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      value: row.value,
+    }))
+  }
+  return rows
+}
+
+const CALCULATION_TYPES = {
+  SUM: "sum",
+  COUNT: "count",
+  STATS: "stats",
+}
+
+async function getView(db: Database, viewName: string) {
+  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
+  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
+  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
+  let viewInfo,
+    migrate = false
+  try {
+    viewInfo = await mainGetter(db, viewName)
+  } catch (err: any) {
+    // check if it can be retrieved from design doc (needs migrated)
+    if (err.status !== 404) {
+      viewInfo = null
+    } else {
+      viewInfo = await secondaryGetter(db, viewName)
+      migrate = !!viewInfo
+    }
+  }
+  if (migrate) {
+    await migration(db, viewName)
+  }
+  if (!viewInfo) {
+    throw "View does not exist."
+  }
+  return viewInfo
+}

packages/server/src/sdk/app/rows/utils.ts

@@ -0,0 +1,48 @@
+import { TableSchema } from "@budibase/types"
+import { FieldTypes } from "../../../constants"
+import { makeExternalQuery } from "../../../integrations/base/query"
+import { Format } from "../../../api/controllers/view/exporters"
+import sdk from "../.."
+
+export async function getDatasourceAndQuery(json: any) {
+  const datasourceId = json.endpoint.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId)
+  return makeExternalQuery(datasource, json)
+}
+
+export function cleanExportRows(
+  rows: any[],
+  schema: TableSchema,
+  format: string,
+  columns?: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  if (format === Format.CSV) {
+    // Intended to append empty values in export
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = undefined
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

packages/server/src/sdk/app/tables/index.ts

@@ -28,7 +28,6 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
 async function getAllExternalTables(
   datasourceId: any
 ): Promise<Record<string, Table>> {
-  const db = context.getAppDB()
   const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -0,0 +1,101 @@
+import { exportRows } from "../../app/rows/search/external"
+import sdk from "../.."
+import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
+import { ExportRowsParams } from "../../app/rows/search"
+import { Format } from "../../../api/controllers/view/exporters"
+import { HTTPError } from "@budibase/backend-core"
+import { Operation } from "@budibase/types"
+
+const mockDatasourcesGet = jest.fn()
+sdk.datasources.get = mockDatasourcesGet
+
+jest.mock("../../../api/controllers/row/ExternalRequest")
+
+jest.mock("../../../api/controllers/view/exporters", () => ({
+  ...jest.requireActual("../../../api/controllers/view/exporters"),
+  csv: jest.fn(),
+  Format: {
+    CSV: "csv",
+  },
+}))
+jest.mock("../../../utilities/fileSystem")
+
+describe("external row sdk", () => {
+  describe("exportRows", () => {
+    function getExportOptions(): ExportRowsParams {
+      return {
+        tableId: "datasource__tablename",
+        format: Format.CSV,
+        query: {},
+      }
+    }
+
+    const externalRequestCall = jest.fn()
+    beforeAll(() => {
+      jest
+        .spyOn(ExternalRequest.prototype, "run")
+        .mockImplementation(externalRequestCall.mockResolvedValue([]))
+    })
+
+    afterEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("should throw a 400 if no datasource entities are present", async () => {
+      const exportOptions = getExportOptions()
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Datasource has not been configured for plus API.", 400)
+      )
+    })
+
+    it("should handle single quotes from a row ID", async () => {
+      mockDatasourcesGet.mockImplementation(async () => ({
+        entities: {
+          tablename: {
+            schema: {},
+          },
+        },
+      }))
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["['d001']"]
+
+      await exportRows(exportOptions)
+
+      expect(ExternalRequest).toBeCalledTimes(1)
+      expect(ExternalRequest).toBeCalledWith(
+        Operation.READ,
+        exportOptions.tableId,
+        undefined
+      )
+
+      expect(externalRequestCall).toBeCalledTimes(1)
+      expect(externalRequestCall).toBeCalledWith(
+        expect.objectContaining({
+          filters: {
+            oneOf: {
+              _id: ["d001"],
+            },
+          },
+        })
+      )
+    })
+
+    it("should throw a 400 if any composite keys are present", async () => {
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"]
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Export data does not support composite keys.", 400)
+      )
+    })
+
+    it("should throw a 400 if no table name was found", async () => {
+      const exportOptions = getExportOptions()
+      exportOptions.tableId = "datasource__"
+      exportOptions.rowIds = ["[123]"]
+
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Could not find table name.", 400)
+      )
+    })
+  })
+})

packages/server/src/sdk/users/crud.ts

@@ -0,0 +1,7 @@
+import { context } from "@budibase/backend-core"
+import { User } from "@budibase/types"
+
+export function get(userId: string) {
+  const db = context.getAppDB()
+  return db.get<User>(userId)
+}

packages/server/src/sdk/users/index.ts

@@ -1,5 +1,9 @@
 import * as utils from "./utils"
+import * as sessions from "./sessions"
+import * as crud from "./crud"
 
 export default {
   ...utils,
+  ...crud,
+  sessions,
 }

packages/server/src/sdk/users/sessions.ts

@@ -0,0 +1,38 @@
+import { builderSocket } from "../../websockets"
+import { App, SocketSession } from "@budibase/types"
+
+export const enrichApps = async (apps: App[]) => {
+  // Sessions can only exist for dev app IDs
+  const devAppIds = apps
+    .filter((app: any) => app.status === "development")
+    .map((app: any) => app.appId)
+
+  // Get all sessions for all apps and enrich app list
+  const sessions = await builderSocket?.getRoomSessions(devAppIds)
+  if (sessions?.length) {
+    let appSessionMap: Record<string, SocketSession[]> = {}
+    sessions.forEach(session => {
+      const room = session.room
+      if (!room) {
+        return
+      }
+      if (!appSessionMap[room]) {
+        appSessionMap[room] = []
+      }
+      appSessionMap[room].push(session)
+    })
+    return apps.map(app => {
+      // Shallow clone to avoid mutating original reference
+      let enriched = { ...app }
+      const sessions = appSessionMap[app.appId]
+      if (sessions?.length) {
+        enriched.sessions = sessions
+      } else {
+        delete enriched.sessions
+      }
+      return enriched
+    })
+  } else {
+    return apps
+  }
+}

packages/server/src/sdk/users/utils.ts

@@ -64,6 +64,25 @@ export async function rawUserMetadata(db?: Database) {
   ).rows.map(row => row.doc)
 }
 
+export async function fetchMetadata() {
+  const global = await getGlobalUsers()
+  const metadata = await rawUserMetadata()
+  const users = []
+  for (let user of global) {
+    // find the metadata that matches up to the global ID
+    const info = metadata.find(meta => meta._id.includes(user._id))
+    // remove these props, not for the correct DB
+    users.push({
+      ...user,
+      ...info,
+      tableId: InternalTables.USER_METADATA,
+      // make sure the ID is always a local ID, not a global one
+      _id: generateUserMetadataID(user._id),
+    })
+  }
+  return users
+}
+
 export async function syncGlobalUsers() {
   // sync user metadata
   const dbs = [context.getDevAppDB(), context.getProdAppDB()]

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -135,7 +135,10 @@ class TestConfiguration {
     }
   }
 
-  async doInContext(appId: string | null, task: any) {
+  async doInContext<T>(
+    appId: string | null,
+    task: () => Promise<T>
+  ): Promise<T> {
     if (!appId) {
       appId = this.appId
     }

packages/server/src/threads/query.ts

@@ -11,6 +11,7 @@ import { cloneDeep } from "lodash/fp"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
+import { Query } from "@budibase/types"
 
 class QueryRunner {
   datasource: any
@@ -167,7 +168,7 @@ class QueryRunner {
 
   async runAnotherQuery(queryId: string, parameters: any) {
     const db = context.getAppDB()
-    const query = await db.get(queryId)
+    const query = await db.get<Query>(queryId)
     const datasource = await sdk.datasources.get(query.datasourceId, {
       enriched: true,
     })

packages/server/src/utilities/global.ts

@@ -90,7 +90,7 @@ export async function getCachedSelf(ctx: UserCtx, appId: string) {
 
 export async function getRawGlobalUser(userId: string) {
   const db = tenancy.getGlobalDB()
-  return db.get(getGlobalIDFromUserMetadataID(userId))
+  return db.get<User>(getGlobalIDFromUserMetadataID(userId))
 }
 
 export async function getGlobalUser(userId: string) {

packages/server/src/utilities/index.ts

@@ -41,7 +41,7 @@ export async function updateEntityMetadata(
   // read it to see if it exists, we'll overwrite it no matter what
   let rev, metadata: Document
   try {
-    const oldMetadata = await db.get(id)
+    const oldMetadata = await db.get<any>(id)
     rev = oldMetadata._rev
     metadata = updateFn(oldMetadata)
   } catch (err) {
@@ -75,7 +75,7 @@ export async function deleteEntityMetadata(type: string, entityId: string) {
   const id = generateMetadataID(type, entityId)
   let rev
   try {
-    const metadata = await db.get(id)
+    const metadata = await db.get<any>(id)
     if (metadata) {
       rev = metadata._rev
     }

packages/server/src/websockets/websocket.ts

@@ -125,9 +125,18 @@ export class BaseSocket {
   }
 
   // Gets an array of all redis keys of users inside a certain room
-  async getRoomSessionIds(room: string): Promise<string[]> {
+  async getRoomSessionIds(room: string | string[]): Promise<string[]> {
-    const keys = await this.redisClient?.get(this.getRoomKey(room))
+    if (Array.isArray(room)) {
-    return keys || []
+      const roomKeys = room.map(this.getRoomKey.bind(this))
+      const roomSessionIdMap = await this.redisClient?.bulkGet(roomKeys)
+      let sessionIds: any[] = []
+      Object.values(roomSessionIdMap || {}).forEach(roomSessionIds => {
+        sessionIds = sessionIds.concat(roomSessionIds)
+      })
+      return sessionIds
+    } else {
+      return (await this.redisClient?.get(this.getRoomKey(room))) || []
+    }
   }
 
   // Sets the list of redis keys for users inside a certain room.
@@ -137,7 +146,7 @@ export class BaseSocket {
   }
 
   // Gets a list of all users inside a certain room
-  async getRoomSessions(room?: string): Promise<SocketSession[]> {
+  async getRoomSessions(room?: string | string[]): Promise<SocketSession[]> {
     if (room) {
       const sessionIds = await this.getRoomSessionIds(room)
       const keys = sessionIds.map(this.getSessionKey.bind(this))

packages/types/src/documents/app/app.ts

@@ -1,4 +1,5 @@
 import { User, Document } from "../"
+import { SocketSession } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
 
@@ -17,6 +18,7 @@ export interface App extends Document {
   customTheme?: AppCustomTheme
   revertableVersion?: string
   lockedBy?: User
+  sessions?: SocketSession[]
   navigation?: AppNavigation
   automationErrors?: AppMetadataErrors
   icon?: AppIcon

packages/types/src/sdk/db.ts

@@ -89,7 +89,7 @@ export interface Database {
 
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
-  get<T>(id?: string): Promise<T | any>
+  get<T>(id?: string): Promise<T>
   remove(
     id: string | Document,
     rev?: string

packages/types/src/sdk/search.ts

@@ -32,7 +32,7 @@ export interface SearchFilters {
     [key: string]: any[]
   }
   contains?: {
-    [key: string]: any[]
+    [key: string]: any[] | any
   }
   notContains?: {
     [key: string]: any[]

packages/worker/src/api/controllers/global/email.ts

@@ -1,6 +1,6 @@
 import { sendEmail as sendEmailFn } from "../../../utilities/email"
 import { tenancy } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { BBContext, User } from "@budibase/types"
 
 export async function sendEmail(ctx: BBContext) {
   let {
@@ -16,10 +16,10 @@ export async function sendEmail(ctx: BBContext) {
     automation,
     invite,
   } = ctx.request.body
-  let user
+  let user: any
   if (userId) {
     const db = tenancy.getGlobalDB()
-    user = await db.get(userId)
+    user = await db.get<User>(userId)
   }
   const response = await sendEmailFn(email, purpose, {
     workspaceId,

packages/worker/src/api/controllers/global/roles.ts

@@ -35,7 +35,7 @@ export async function find(ctx: BBContext) {
   const appId = ctx.params.appId
   await context.doInAppContext(dbCore.getDevAppID(appId), async () => {
     const db = context.getAppDB()
-    const app = await db.get(dbCore.DocumentType.APP_METADATA)
+    const app = await db.get<App>(dbCore.DocumentType.APP_METADATA)
     ctx.body = {
       roles: await roles.getAllRoles(),
       name: app.name,

packages/worker/src/api/controllers/global/self.ts

@@ -44,7 +44,7 @@ export async function generateAPIKey(ctx: any) {
   const id = dbCore.generateDevInfoID(userId)
   let devInfo
   try {
-    devInfo = await db.get(id)
+    devInfo = await db.get<any>(id)
   } catch (err) {
     devInfo = { _id: id, userId }
   }

packages/worker/src/api/controllers/global/users.ts

@@ -414,7 +414,7 @@ export const inviteAccept = async (
 
       const saved = await userSdk.save(request)
       const db = tenancy.getGlobalDB()
-      const user = await db.get(saved._id)
+      const user = await db.get<User>(saved._id)
       await events.user.inviteAccepted(user)
       return saved
     })