Merge remote-tracking branch 'origin/develop' into feature/skippable-tours

.github/workflows/budibase_ci.yml

@@ -12,9 +12,6 @@ on:
       - master
       - develop
   pull_request:
-    branches:
-      - master
-      - develop
   workflow_dispatch:
 
 env:

.tool-versions

@@ -1,2 +1,2 @@
-nodejs 14.20.1
+nodejs 14.21.3
 python 3.10.0

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.10-alpha.4",
+  "version": "2.8.12-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -53,7 +53,7 @@
     "kill-all": "yarn run kill-builder && yarn run kill-server",
     "dev": "yarn run kill-all && lerna run --stream dev:builder --stream",
     "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && yarn build --projects=@budibase/client && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
+    "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/bbui/src/Form/Core/Combobox.svelte

@@ -82,7 +82,7 @@
   {#if open}
     <div class="overlay" on:mousedown|self={() => (open = false)} />
     <div
-      transition:fly={{ y: -20, duration: 200 }}
+      transition:fly|local={{ y: -20, duration: 200 }}
       class="spectrum-Popover spectrum-Popover--bottom is-open"
     >
       <ul class="spectrum-Menu" role="listbox">

packages/builder/src/components/backend/TableNavigator/popovers/EditViewPopover.svelte

@@ -35,9 +35,8 @@
     try {
       const isSelected =
         decodeURIComponent($params.viewName) === $views.selectedViewName
-      const name = view.name
       const id = view.tableId
-      await views.delete(name)
+      await views.delete(view)
       notifications.success("View deleted")
       if (isSelected) {
         $goto(`./table/${id}`)

packages/builder/src/components/common/NavItem.svelte

@@ -2,7 +2,7 @@
   import { Icon } from "@budibase/bbui"
   import { createEventDispatcher, getContext } from "svelte"
   import { helpers } from "@budibase/shared-core"
-  import UserAvatars from "../../pages/builder/app/[application]/_components/UserAvatars.svelte"
+  import { UserAvatars } from "@budibase/frontend-core"
 
   export let icon
   export let withArrow = false

packages/builder/src/components/start/AppRow.svelte

@@ -2,12 +2,12 @@
   import { Heading, Body, Button, Icon } from "@budibase/bbui"
   import { processStringSync } from "@budibase/string-templates"
   import { goto } from "@roxi/routify"
-  import { UserAvatar } from "@budibase/frontend-core"
+  import { UserAvatars } from "@budibase/frontend-core"
 
   export let app
   export let lockedAction
 
-  $: editing = app?.lockedBy != null
+  $: editing = app.sessions?.length
 
   const handleDefaultClick = () => {
     if (window.innerWidth < 640) {
@@ -41,7 +41,7 @@
   <div class="updated">
     {#if editing}
       Currently editing
-      <UserAvatar user={app.lockedBy} />
+      <UserAvatars users={app.sessions} />
     {:else if app.updatedAt}
       {processStringSync("Updated {{ duration time 'millisecond' }} ago", {
         time: new Date().getTime() - new Date(app.updatedAt).getTime(),

packages/builder/src/stores/backend/views.js

@@ -26,14 +26,12 @@ export function createViewsStore() {
   }
 
   const deleteView = async view => {
-    await API.deleteView(view)
+    await API.deleteView(view.name)
 
     // Update tables
     tables.update(state => {
       const table = state.list.find(table => table._id === view.tableId)
-      if (table) {
+      delete table.views[view.name]
-        delete table.views[view.name]
-      }
       return { ...state }
     })
   }

packages/frontend-core/src/components/index.js

@@ -2,4 +2,5 @@ export { default as SplitPage } from "./SplitPage.svelte"
 export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
+export { default as UserAvatars } from "./UserAvatars.svelte"
 export { Grid } from "./grid"

packages/server/src/api/controllers/application.ts

@@ -49,6 +49,7 @@ import {
   MigrationType,
   PlanType,
   Screen,
+  SocketSession,
   UserCtx,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
@@ -183,6 +184,7 @@ export async function fetch(ctx: UserCtx) {
   const appIds = apps
     .filter((app: any) => app.status === "development")
     .map((app: any) => app.appId)
+
   // get the locks for all the dev apps
   if (dev || all) {
     const locks = await getLocksById(appIds)
@@ -197,7 +199,10 @@ export async function fetch(ctx: UserCtx) {
     }
   }
 
-  ctx.body = await checkAppMetadata(apps)
+  // Enrich apps with all builder user sessions
+  const enrichedApps = await sdk.users.sessions.enrichApps(apps)
+
+  ctx.body = await checkAppMetadata(enrichedApps)
 }
 
 export async function fetchAppDefinition(ctx: UserCtx) {

packages/server/src/api/controllers/datasource.ts

@@ -9,7 +9,6 @@ import {
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
-import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
 import {
@@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) {
 export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await getDatasourceAndQuery(queryJson)
+    ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
   } catch (err: any) {
     ctx.throw(400, err)
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -23,14 +23,13 @@ import {
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
-import { getDatasourceAndQuery } from "./utils"
+import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { isEditableColumn } from "../../../sdk/app/tables/validation"
 
 export interface ManyRelationship {
   tableId?: string

packages/server/src/api/controllers/row/external.ts

@@ -1,30 +1,20 @@
-import {
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-  FieldTypes,
-  NoEmptyFilterStrings,
-  SortDirection,
-} from "../../../constants"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../integrations/utils"
 import { ExternalRequest, RunConfig } from "./ExternalRequest"
-import * as exporters from "../view/exporters"
-import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Datasource,
   IncludeRelationship,
   Operation,
-  PaginationJson,
   Row,
-  SortJson,
   Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 
-const { cleanExportRows } = require("./utils")
-
 async function getRow(
   tableId: string,
   rowId: string,
@@ -59,6 +49,7 @@ export async function handleRequest(
       }
     }
   }
+
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
     opts || {}
   )
@@ -114,21 +105,6 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function fetchView(ctx: UserCtx) {
-  // there are no views in external datasources, shouldn't ever be called
-  // for now just fetch
-  const split = ctx.params.viewName.split("all_")
-  ctx.params.tableId = split[1] ? split[1] : split[0]
-  return fetch(ctx)
-}
-
-export async function fetch(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  return handleRequest(Operation.READ, tableId, {
-    includeSqlRelationships: IncludeRelationship.INCLUDE,
-  })
-}
-
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
@@ -161,129 +137,6 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: UserCtx) {
-  const tableId = ctx.params.tableId
-  const { paginate, query, ...params } = ctx.request.body
-  let { bookmark, limit } = params
-  if (!bookmark && paginate) {
-    bookmark = 1
-  }
-  let paginateObj = {}
-
-  if (paginate) {
-    paginateObj = {
-      // add one so we can track if there is another page
-      limit: limit,
-      page: bookmark,
-    }
-  } else if (params && limit) {
-    paginateObj = {
-      limit: limit,
-    }
-  }
-  let sort: SortJson | undefined
-  if (params.sort) {
-    const direction =
-      params.sortOrder === "descending"
-        ? SortDirection.DESCENDING
-        : SortDirection.ASCENDING
-    sort = {
-      [params.sort]: { direction },
-    }
-  }
-  try {
-    const rows = (await handleRequest(Operation.READ, tableId, {
-      filters: query,
-      sort,
-      paginate: paginateObj as PaginationJson,
-      includeSqlRelationships: IncludeRelationship.INCLUDE,
-    })) as Row[]
-    let hasNextPage = false
-    if (paginate && rows.length === limit) {
-      const nextRows = (await handleRequest(Operation.READ, tableId, {
-        filters: query,
-        sort,
-        paginate: {
-          limit: 1,
-          page: bookmark * limit + 1,
-        },
-        includeSqlRelationships: IncludeRelationship.INCLUDE,
-      })) as Row[]
-      hasNextPage = nextRows.length > 0
-    }
-    // need wrapper object for bookmarks etc when paginating
-    return { rows, hasNextPage, bookmark: bookmark + 1 }
-  } catch (err: any) {
-    if (err.message && err.message.includes("does not exist")) {
-      throw new Error(
-        `Table updated externally, please re-fetch - ${err.message}`
-      )
-    } else {
-      throw err
-    }
-  }
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
-  const format = ctx.query.format
-  const { columns } = ctx.request.body
-  const datasource = await sdk.datasources.get(datasourceId!)
-  if (!datasource || !datasource.entities) {
-    ctx.throw(400, "Datasource has not been configured for plus API.")
-  }
-
-  if (ctx.request.body.rows) {
-    ctx.request.body = {
-      query: {
-        oneOf: {
-          _id: ctx.request.body.rows.map((row: string) => {
-            const ids = JSON.parse(
-              decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
-            )
-            if (ids.length > 1) {
-              ctx.throw(400, "Export data does not support composite keys.")
-            }
-            return ids[0]
-          }),
-        },
-      },
-    }
-  }
-
-  let result = await search(ctx)
-  let rows: Row[] = []
-
-  // Filter data to only specified columns if required
-
-  if (columns && columns.length) {
-    for (let i = 0; i < result.rows.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result.rows[i][column]
-      }
-    }
-  } else {
-    rows = result.rows
-  }
-
-  if (!tableName) {
-    ctx.throw(400, "Could not find table name.")
-  }
-  let schema = datasource.entities[tableName].schema
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-
-  let headers = Object.keys(schema)
-
-  // @ts-ignore
-  const exporter = exporters[format]
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, exportRows))
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId

packages/server/src/api/controllers/row/index.ts

@@ -5,6 +5,9 @@ import { isExternalTable } from "../../../integrations/utils"
 import { Ctx } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
+import sdk from "../../../sdk"
+import * as exporters from "../view/exporters"
+import { apiFileReturn } from "../../../utilities/fileSystem"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -64,14 +67,26 @@ export const save = async (ctx: any) => {
 }
 export async function fetchView(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
+  const viewName = decodeURIComponent(ctx.params.viewName)
-    datasourceId: tableId,
+
-  })
+  const { calculation, group, field } = ctx.query
+
+  ctx.body = await quotas.addQuery(
+    () =>
+      sdk.rows.fetchView(tableId, viewName, {
+        calculation,
+        group,
+        field,
+      }),
+    {
+      datasourceId: tableId,
+    }
+  )
 }
 
 export async function fetch(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
     datasourceId: tableId,
   })
 }
@@ -119,8 +134,14 @@ export async function destroy(ctx: any) {
 
 export async function search(ctx: any) {
   const tableId = utils.getTableId(ctx)
+
+  const searchParams = {
+    ...ctx.request.body,
+    tableId,
+  }
+
   ctx.status = 200
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
     datasourceId: tableId,
   })
 }
@@ -150,7 +171,33 @@ export async function fetchEnrichedRow(ctx: any) {
 
 export const exportRows = async (ctx: any) => {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
+
-    datasourceId: tableId,
+  const format = ctx.query.format
-  })
+
+  const { rows, columns, query } = ctx.request.body
+  if (typeof format !== "string" || !exporters.isFormat(format)) {
+    ctx.throw(
+      400,
+      `Format ${format} not valid. Valid values: ${Object.values(
+        exporters.Format
+      )}`
+    )
+  }
+
+  ctx.body = await quotas.addQuery(
+    async () => {
+      const { fileName, content } = await sdk.rows.exportRows({
+        tableId,
+        format,
+        rowIds: rows,
+        columns,
+        query,
+      })
+      ctx.attachment(fileName)
+      return apiFileReturn(content)
+    },
+    {
+      datasourceId: tableId,
+    }
+  )
 }

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,7 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
-  getRowParams,
   getTableIDFromRowID,
-  DocumentType,
   InternalTables,
 } from "../../../db/utils"
 import * as userController from "../user"
@@ -14,78 +12,10 @@ import {
 } from "../../../utilities/rowProcessor"
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
-import { fullSearch, paginatedSearch } from "./internalSearch"
-import { getGlobalUsersFromMetadata } from "../../../utilities/global"
-import * as inMemoryViews from "../../../db/inMemoryView"
-import env from "../../../environment"
-import {
-  migrateToInMemoryView,
-  migrateToDesignView,
-  getFromDesignDoc,
-  getFromMemoryDoc,
-} from "../view/utils"
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format } from "../view/exporters"
+import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
-import { apiFileReturn } from "../../../utilities/fileSystem"
-import {
-  UserCtx,
-  Database,
-  LinkDocumentValue,
-  Row,
-  Table,
-} from "@budibase/types"
-
-import { cleanExportRows } from "./utils"
-
-const CALCULATION_TYPES = {
-  SUM: "sum",
-  COUNT: "count",
-  STATS: "stats",
-}
-
-async function getView(db: Database, viewName: string) {
-  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
-  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
-  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
-  let viewInfo,
-    migrate = false
-  try {
-    viewInfo = await mainGetter(db, viewName)
-  } catch (err: any) {
-    // check if it can be retrieved from design doc (needs migrated)
-    if (err.status !== 404) {
-      viewInfo = null
-    } else {
-      viewInfo = await secondaryGetter(db, viewName)
-      migrate = !!viewInfo
-    }
-  }
-  if (migrate) {
-    await migration(db, viewName)
-  }
-  if (!viewInfo) {
-    throw "View does not exist."
-  }
-  return viewInfo
-}
-
-async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
-  let rows
-  if (tableId === InternalTables.USER_METADATA) {
-    await userController.fetchMetadata(ctx)
-    rows = ctx.body
-  } else {
-    const response = await db.allDocs(
-      getRowParams(tableId, null, {
-        include_docs: true,
-      })
-    )
-    rows = response.rows.map(row => row.doc)
-  }
-  return rows as Row[]
-}
 
 export async function patch(ctx: UserCtx) {
   const db = context.getAppDB()
@@ -195,82 +125,6 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: UserCtx) {
-  const viewName = decodeURIComponent(ctx.params.viewName)
-
-  // if this is a table view being looked for just transfer to that
-  if (viewName.startsWith(DocumentType.TABLE)) {
-    ctx.params.tableId = viewName
-    return fetch(ctx)
-  }
-
-  const db = context.getAppDB()
-  const { calculation, group, field } = ctx.query
-  const viewInfo = await getView(db, viewName)
-  let response
-  if (env.SELF_HOSTED) {
-    response = await db.query(`database/${viewName}`, {
-      include_docs: !calculation,
-      group: !!group,
-    })
-  } else {
-    const tableId = viewInfo.meta.tableId
-    const data = await getRawTableData(ctx, db, tableId)
-    response = await inMemoryViews.runView(
-      viewInfo,
-      calculation as string,
-      !!group,
-      data
-    )
-  }
-
-  let rows
-  if (!calculation) {
-    response.rows = response.rows.map(row => row.doc)
-    let table
-    try {
-      table = await db.get(viewInfo.meta.tableId)
-    } catch (err) {
-      /* istanbul ignore next */
-      table = {
-        schema: {},
-      }
-    }
-    rows = await outputProcessing(table, response.rows)
-  }
-
-  if (calculation === CALCULATION_TYPES.STATS) {
-    response.rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      ...row.value,
-      avg: row.value.sum / row.value.count,
-    }))
-    rows = response.rows
-  }
-
-  if (
-    calculation === CALCULATION_TYPES.COUNT ||
-    calculation === CALCULATION_TYPES.SUM
-  ) {
-    rows = response.rows.map(row => ({
-      group: row.key,
-      field,
-      value: row.value,
-    }))
-  }
-  return rows
-}
-
-export async function fetch(ctx: UserCtx) {
-  const db = context.getAppDB()
-
-  const tableId = ctx.params.tableId
-  let table = await db.get(tableId)
-  let rows = await getRawTableData(ctx, db, tableId)
-  return outputProcessing(table, rows)
-}
-
 export async function find(ctx: UserCtx) {
   const db = dbCore.getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
@@ -354,101 +208,6 @@ export async function bulkDestroy(ctx: UserCtx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: UserCtx) {
-  // Fetch the whole table when running in cypress, as search doesn't work
-  if (!env.COUCH_DB_URL && env.isCypress()) {
-    return { rows: await fetch(ctx) }
-  }
-
-  const { tableId } = ctx.params
-  const db = context.getAppDB()
-  const { paginate, query, ...params } = ctx.request.body
-  params.version = ctx.version
-  params.tableId = tableId
-
-  let table
-  if (params.sort && !params.sortType) {
-    table = await db.get(tableId)
-    const schema = table.schema
-    const sortField = schema[params.sort]
-    params.sortType = sortField.type == "number" ? "number" : "string"
-  }
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(query, params)
-  } else {
-    response = await fullSearch(query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    // enrich with global users if from users table
-    if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows)
-    }
-    table = table || (await db.get(tableId))
-    response.rows = await outputProcessing(table, response.rows)
-  }
-
-  return response
-}
-
-export async function exportRows(ctx: UserCtx) {
-  const db = context.getAppDB()
-  const table = await db.get(ctx.params.tableId)
-  const rowIds = ctx.request.body.rows
-  let format = ctx.query.format
-  if (typeof format !== "string") {
-    ctx.throw(400, "Format parameter is not valid")
-  }
-  const { columns, query } = ctx.request.body
-
-  let result
-  if (rowIds) {
-    let response = (
-      await db.allDocs({
-        include_docs: true,
-        keys: rowIds,
-      })
-    ).rows.map(row => row.doc)
-
-    result = await outputProcessing(table, response)
-  } else if (query) {
-    let searchResponse = await search(ctx)
-    result = searchResponse.rows
-  }
-
-  let rows: Row[] = []
-  let schema = table.schema
-
-  // Filter data to only specified columns if required
-  if (columns && columns.length) {
-    for (let i = 0; i < result.length; i++) {
-      rows[i] = {}
-      for (let column of columns) {
-        rows[i][column] = result[i][column]
-      }
-    }
-  } else {
-    rows = result
-  }
-
-  let exportRows = cleanExportRows(rows, schema, format, columns)
-  if (format === Format.CSV) {
-    ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
-  } else if (format === Format.JSON) {
-    ctx.attachment("export.json")
-    return apiFileReturn(json(exportRows))
-  } else if (format === Format.JSON_WITH_SCHEMA) {
-    ctx.attachment("export.json")
-    return apiFileReturn(jsonWithSchema(schema, exportRows))
-  } else {
-    throw "Format not recognised"
-  }
-}
-
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId

packages/server/src/api/controllers/row/utils.ts

@@ -1,14 +1,19 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
-import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
-import { makeExternalQuery } from "../../../integrations/base/query"
+import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldType, Row, Table, UserCtx } from "@budibase/types"
+import { FieldTypes } from "../../../constants"
-import { Format } from "../view/exporters"
 import sdk from "../../../sdk"
 
-const validateJs = require("validate.js")
+import validateJs from "validate.js"
-const { cloneDeep } = require("lodash/fp")
+import { cloneDeep } from "lodash/fp"
+
+function isForeignKey(key: string, table: Table) {
+  const relationships = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  return relationships.some(relationship => relationship.foreignKey === key)
+}
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
-  )
-  return relationships.some(relationship => relationship.foreignKey === key)
-}
-
-export async function getDatasourceAndQuery(json: any) {
-  const datasourceId = json.endpoint.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId)
-  return makeExternalQuery(datasource, json)
-}
-
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
@@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   return row
 }
 
+export function getTableId(ctx: Ctx) {
+  if (ctx.request.body && ctx.request.body.tableId) {
+    return ctx.request.body.tableId
+  }
+  if (ctx.params && ctx.params.tableId) {
+    return ctx.params.tableId
+  }
+  if (ctx.params && ctx.params.viewName) {
+    return ctx.params.viewName
+  }
+}
+
 export async function validate({
   tableId,
   row,
@@ -81,8 +85,8 @@ export async function validate({
       continue
     }
     // special case for options, need to always allow unselected (empty)
-    if (type === FieldTypes.OPTIONS && constraints.inclusion) {
+    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
-      constraints.inclusion.push(null, "")
+      constraints.inclusion.push(null as any, "")
     }
     let res
 
@@ -94,13 +98,13 @@ export async function validate({
         }
         row[fieldName].map((val: any) => {
           if (
-            !constraints.inclusion.includes(val) &&
+            !constraints?.inclusion?.includes(val) &&
-            constraints.inclusion.length !== 0
+            constraints?.inclusion?.length !== 0
           ) {
             errors[fieldName] = "Field not in list"
           }
         })
-      } else if (constraints.presence && row[fieldName].length === 0) {
+      } else if (constraints?.presence && row[fieldName].length === 0) {
         // non required MultiSelect creates an empty array, which should not throw errors
         errors[fieldName] = [`${fieldName} is required`]
       }
@@ -128,52 +132,3 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
-
-export function cleanExportRows(
-  rows: any[],
-  schema: any,
-  format: string,
-  columns: string[]
-) {
-  let cleanRows = [...rows]
-
-  const relationships = Object.entries(schema)
-    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-
-  relationships.forEach(column => {
-    cleanRows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  if (format === Format.CSV) {
-    // Intended to append empty values in export
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      if (columns?.length && columns.indexOf(key) > 0) {
-        continue
-      }
-      for (let row of cleanRows) {
-        if (row[key] == null) {
-          row[key] = undefined
-        }
-      }
-    }
-  }
-
-  return cleanRows
-}
-
-export function getTableId(ctx: any) {
-  if (ctx.request.body && ctx.request.body.tableId) {
-    return ctx.request.body.tableId
-  }
-  if (ctx.params && ctx.params.tableId) {
-    return ctx.params.tableId
-  }
-  if (ctx.params && ctx.params.viewName) {
-    return ctx.params.viewName
-  }
-}

packages/server/src/api/controllers/tests/row.spec.ts

@@ -1,120 +0,0 @@
-import { exportRows } from "../row/external"
-import sdk from "../../../sdk"
-import { ExternalRequest } from "../row/ExternalRequest"
-
-// @ts-ignore
-sdk.datasources = {
-  get: jest.fn(),
-}
-
-jest.mock("../row/ExternalRequest")
-jest.mock("../view/exporters", () => ({
-  csv: jest.fn(),
-  Format: {
-    CSV: "csv",
-  },
-}))
-jest.mock("../../../utilities/fileSystem")
-
-function getUserCtx() {
-  return {
-    params: {
-      tableId: "datasource__tablename",
-    },
-    query: {
-      format: "csv",
-    },
-    request: {
-      body: {},
-    },
-    throw: jest.fn(() => {
-      throw "Err"
-    }),
-    attachment: jest.fn(),
-  }
-}
-
-describe("external row controller", () => {
-  describe("exportRows", () => {
-    beforeAll(() => {
-      //@ts-ignore
-      jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
-    })
-
-    afterEach(() => {
-      jest.clearAllMocks()
-    })
-
-    it("should throw a 400 if no datasource entities are present", async () => {
-      let userCtx = getUserCtx()
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Datasource has not been configured for plus API."
-        )
-      }
-    })
-
-    it("should handle single quotes from a row ID", async () => {
-      //@ts-ignore
-      sdk.datasources.get.mockImplementation(() => ({
-        entities: {
-          tablename: {
-            schema: {},
-          },
-        },
-      }))
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["['d001']"],
-      }
-
-      //@ts-ignore
-      await exportRows(userCtx)
-
-      expect(userCtx.request.body).toEqual({
-        query: {
-          oneOf: {
-            _id: ["d001"],
-          },
-        },
-      })
-    })
-
-    it("should throw a 400 if any composite keys are present", async () => {
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["[123]", "['d001'%2C'10111']"],
-      }
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Export data does not support composite keys."
-        )
-      }
-    })
-
-    it("should throw a 400 if no table name was found", async () => {
-      let userCtx = getUserCtx()
-      userCtx.params.tableId = "datasource__"
-      userCtx.request.body = {
-        rows: ["[123]"],
-      }
-      try {
-        //@ts-ignore
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Could not find table name."
-        )
-      }
-    })
-  })
-})

packages/server/src/api/controllers/user.ts

@@ -3,25 +3,11 @@ import { InternalTables } from "../../db/utils"
 import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function fetchMetadata(ctx: UserCtx) {
+export async function fetchMetadata(ctx: Ctx) {
-  const global = await getGlobalUsers()
+  const users = await sdk.users.fetchMetadata()
-  const metadata = await sdk.users.rawUserMetadata()
-  const users = []
-  for (let user of global) {
-    // find the metadata that matches up to the global ID
-    const info = metadata.find(meta => meta._id.includes(user._id))
-    // remove these props, not for the correct DB
-    users.push({
-      ...user,
-      ...info,
-      tableId: InternalTables.USER_METADATA,
-      // make sure the ID is always a local ID, not a global one
-      _id: generateUserMetadataID(user._id),
-    })
-  }
   ctx.body = users
 }
 

packages/server/src/api/controllers/view/index.ts

@@ -15,7 +15,6 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
-import { cleanExportRows } from "../row/utils"
 import { builderSocket } from "../../../websockets"
 
 const { cloneDeep, isEqual } = require("lodash")
@@ -163,13 +162,16 @@ export async function exportView(ctx: Ctx) {
   let rows = ctx.body as Row[]
 
   let schema: TableSchema = view && view.meta && view.meta.schema
-  const tableId = ctx.params.tableId || view.meta.tableId
+  const tableId =
+    ctx.params.tableId ||
+    view?.meta?.tableId ||
+    (viewName.startsWith(DocumentType.TABLE) && viewName)
   const table: Table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, [])
+  let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)

packages/server/src/api/routes/tests/internalSearch.spec.js

@@ -1,197 +0,0 @@
-const fetch = require("node-fetch")
-fetch.mockSearch()
-const search = require("../../controllers/row/internalSearch")
-// this will be mocked out for _search endpoint
-const PARAMS = {
-  tableId: "ta_12345679abcdef",
-  version: "1",
-  bookmark: null,
-  sort: null,
-  sortOrder: "ascending",
-  sortType: "string",
-}
-
-function checkLucene(resp, expected, params = PARAMS) {
-  const query = resp.rows[0].query
-  const json = JSON.parse(query)
-  if (PARAMS.sort) {
-    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-  }
-  if (PARAMS.bookmark) {
-    expect(json.bookmark).toBe(PARAMS.bookmark)
-  }
-  expect(json.include_docs).toBe(true)
-  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
-  expect(json.limit).toBe(params.limit || 50)
-}
-
-describe("internal search", () => {
-  it("default query", async () => {
-    const response = await search.paginatedSearch({
-    }, PARAMS)
-    checkLucene(response, `*:*`)
-  })
-
-  it("test equal query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:"1"`)
-  })
-
-  it("test notEqual query", async () => {
-    const response = await search.paginatedSearch({
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `*:* AND !column:"1"`)
-  })
-
-  it("test OR query", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(column:"2" OR !column:"1")`)
-  })
-
-  it("test AND query", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "2",
-      },
-      notEqual: {
-        "column": "1",
-      }
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
-  })
-
-  it("test pagination query", async () => {
-    const updatedParams = {
-      ...PARAMS,
-      limit: 100,
-      bookmark: "awd",
-      sort: "column",
-    }
-    const response = await search.paginatedSearch({
-      string: {
-        "column": "2",
-      },
-    }, updatedParams)
-    checkLucene(response, `*:* AND column:2*`, updatedParams)
-  })
-
-  it("test range query", async () => {
-    const response = await search.paginatedSearch({
-      range: {
-        "column": { low: 1, high: 2 },
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
-  })
-
-  it("test empty query", async () => {
-    const response = await search.paginatedSearch({
-      empty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
-  })
-
-  it("test notEmpty query", async () => {
-    const response = await search.paginatedSearch({
-      notEmpty: {
-        "column": "",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
-  })
-
-  it("test oneOf query", async () => {
-    const response = await search.paginatedSearch({
-      oneOf: {
-        "column": ["a", "b"],
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
-  })
-
-  it("test contains query", async () => {
-    const response = await search.paginatedSearch({
-      contains: {
-        "column": "a",
-        "colArr": [1, 2, 3],
-      },
-    }, PARAMS)
-    checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
-  })
-
-  it("test multiple of same column", async () => {
-    const response = await search.paginatedSearch({
-      allOr: true,
-      equal: {
-        "1:column": "a",
-        "2:column": "b",
-        "3:column": "c",
-      },
-    }, PARAMS)
-    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
-  })
-
-  it("check a weird case for lucene building", async () => {
-    const response = await search.paginatedSearch({
-      equal: {
-        "1:1:column": "a",
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
-  })
-
-  it("test containsAny query", async () => {
-    const response = await search.paginatedSearch({
-      containsAny: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
-  })
-
-  it("test notContains query", async () => {
-    const response = await search.paginatedSearch({
-      notContains: {
-        "column": ["a", "b", "c"]
-      },
-    }, PARAMS)
-    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
-  })
-
-  it("test equal without version query", async () => {
-    PARAMS.version = null
-    const response = await search.paginatedSearch({
-      equal: {
-        "column": "1",
-      }
-    }, PARAMS)
-    
-    const query = response.rows[0].query
-    const json = JSON.parse(query)
-    if (PARAMS.sort) {
-      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
-    }
-    if (PARAMS.bookmark) {
-      expect(json.bookmark).toBe(PARAMS.bookmark)
-    }
-    expect(json.include_docs).toBe(true)
-    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
-  })
-})

packages/server/src/api/routes/tests/internalSearch.spec.ts

@@ -0,0 +1,248 @@
+const nodeFetch = require("node-fetch")
+nodeFetch.mockSearch()
+import { SearchParams } from "@budibase/backend-core"
+import * as search from "../../../sdk/app/rows/search/internalSearch"
+import { Row } from "@budibase/types"
+
+// this will be mocked out for _search endpoint
+const PARAMS: SearchParams<Row> = {
+  tableId: "ta_12345679abcdef",
+  version: "1",
+  bookmark: undefined,
+  sort: undefined,
+  sortOrder: "ascending",
+  sortType: "string",
+}
+
+function checkLucene(resp: any, expected: any, params = PARAMS) {
+  const query = resp.rows[0].query
+  const json = JSON.parse(query)
+  if (PARAMS.sort) {
+    expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+  }
+  if (PARAMS.bookmark) {
+    expect(json.bookmark).toBe(PARAMS.bookmark)
+  }
+  expect(json.include_docs).toBe(true)
+  expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
+  expect(json.limit).toBe(params.limit || 50)
+}
+
+describe("internal search", () => {
+  it("default query", async () => {
+    const response = await search.paginatedSearch({}, PARAMS)
+    checkLucene(response, `*:*`)
+  })
+
+  it("test equal query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:"1"`)
+  })
+
+  it("test notEqual query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND !column:"1"`)
+  })
+
+  it("test OR query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"2" OR !column:"1")`)
+  })
+
+  it("test AND query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "2",
+        },
+        notEqual: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
+  })
+
+  it("test pagination query", async () => {
+    const updatedParams = {
+      ...PARAMS,
+      limit: 100,
+      bookmark: "awd",
+      sort: "column",
+    }
+    const response = await search.paginatedSearch(
+      {
+        string: {
+          column: "2",
+        },
+      },
+      updatedParams
+    )
+    checkLucene(response, `*:* AND column:2*`, updatedParams)
+  })
+
+  it("test range query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        range: {
+          column: { low: 1, high: 2 },
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
+  })
+
+  it("test empty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        empty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
+  })
+
+  it("test notEmpty query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notEmpty: {
+          column: "",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
+  })
+
+  it("test oneOf query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        oneOf: {
+          column: ["a", "b"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
+  })
+
+  it("test contains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        contains: {
+          column: "a",
+          colArr: [1, 2, 3],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(
+      response,
+      `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
+      PARAMS
+    )
+  })
+
+  it("test multiple of same column", async () => {
+    const response = await search.paginatedSearch(
+      {
+        allOr: true,
+        equal: {
+          "1:column": "a",
+          "2:column": "b",
+          "3:column": "c",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
+  })
+
+  it("check a weird case for lucene building", async () => {
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          "1:1:column": "a",
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
+  })
+
+  it("test containsAny query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        containsAny: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
+  })
+
+  it("test notContains query", async () => {
+    const response = await search.paginatedSearch(
+      {
+        notContains: {
+          column: ["a", "b", "c"],
+        },
+      },
+      PARAMS
+    )
+    checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
+  })
+
+  it("test equal without version query", async () => {
+    PARAMS.version = undefined
+    const response = await search.paginatedSearch(
+      {
+        equal: {
+          column: "1",
+        },
+      },
+      PARAMS
+    )
+
+    const query = response.rows[0].query
+    const json = JSON.parse(query)
+    if (PARAMS.sort) {
+      expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
+    }
+    if (PARAMS.bookmark) {
+      expect(json.bookmark).toBe(PARAMS.bookmark)
+    }
+    expect(json.include_docs).toBe(true)
+    expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
+  })
+})

packages/server/src/api/routes/tests/row.spec.ts

@@ -1,27 +1,27 @@
-const tk = require( "timekeeper")
+import tk from "timekeeper"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
-
+import { outputProcessing } from "../../../utilities/rowProcessor"
-const { outputProcessing } = require("../../../utilities/rowProcessor")
+import * as setup from "./utilities"
-const setup = require("./utilities")
 const { basicRow } = setup.structures
-const { context, tenancy } = require("@budibase/backend-core")
+import { context, tenancy } from "@budibase/backend-core"
-const {
+import { quotas } from "@budibase/pro"
-  quotas,
+import {
-} = require("@budibase/pro")
-const {
   QuotaUsageType,
   StaticQuotaName,
   MonthlyQuotaName,
-} = require("@budibase/types")
+  Row,
-const { structures } = require("@budibase/backend-core/tests");
+  Table,
+  FieldType,
+} from "@budibase/types"
+import { structures } from "@budibase/backend-core/tests"
 
 describe("/rows", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
-  let table
+  let table: Table
-  let row
+  let row: Row
 
   afterAll(setup.afterAll)
 
@@ -29,12 +29,12 @@ describe("/rows", () => {
     await config.init()
   })
 
-  beforeEach(async()=>{
+  beforeEach(async () => {
     table = await config.createTable()
-    row = basicRow(table._id)
+    row = basicRow(table._id!)
   })
 
-  const loadRow = async (id, tbl_Id, status = 200) =>
+  const loadRow = async (id: string, tbl_Id: string, status = 200) =>
     await request
       .get(`/api/${tbl_Id}/rows/${id}`)
       .set(config.defaultHeaders())
@@ -42,21 +42,28 @@ describe("/rows", () => {
       .expect(status)
 
   const getRowUsage = async () => {
-    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+    const { total } = await config.doInContext(null, () =>
+      quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
+    )
     return total
   }
 
   const getQueryUsage = async () => {
-    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+    const { total } = await config.doInContext(null, () =>
+      quotas.getCurrentUsageValues(
+        QuotaUsageType.MONTHLY,
+        MonthlyQuotaName.QUERIES
+      )
+    )
     return total
   }
 
-  const assertRowUsage = async expected => {
+  const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
     expect(usage).toBe(expected)
   }
 
-  const assertQueryUsage = async expected => {
+  const assertQueryUsage = async (expected: number) => {
     const usage = await getQueryUsage()
     expect(usage).toBe(expected)
   }
@@ -70,9 +77,11 @@ describe("/rows", () => {
         .post(`/api/${row.tableId}/rows`)
         .send(row)
         .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
+        .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
+      expect((res as any).res.statusMessage).toEqual(
+        `${table.name} saved successfully`
+      )
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
       await assertRowUsage(rowUsage + 1)
@@ -86,12 +95,11 @@ describe("/rows", () => {
       const newTable = await config.createTable({
         name: "TestTableAuto",
         type: "table",
-        key: "name",
         schema: {
           ...table.schema,
           "Row ID": {
             name: "Row ID",
-            type: "number",
+            type: FieldType.NUMBER,
             subtype: "autoID",
             icon: "ri-magic-line",
             autocolumn: true,
@@ -104,28 +112,30 @@ describe("/rows", () => {
               },
             },
           },
-        }
+        },
       })
 
-      const ids = [1,2,3]
+      const ids = [1, 2, 3]
 
       // Performing several create row requests should increment the autoID fields accordingly
-      const createRow = async (id) => {
+      const createRow = async (id: number) => {
         const res = await request
           .post(`/api/${newTable._id}/rows`)
           .send({
-            name: "row_" + id
+            name: "row_" + id,
           })
           .set(config.defaultHeaders())
-          .expect('Content-Type', /json/)
+          .expect("Content-Type", /json/)
           .expect(200)
-        expect(res.res.statusMessage).toEqual(`${newTable.name} saved successfully`)
+        expect((res as any).res.statusMessage).toEqual(
+          `${newTable.name} saved successfully`
+        )
         expect(res.body.name).toEqual("row_" + id)
         expect(res.body._rev).toBeDefined()
         expect(res.body["Row ID"]).toEqual(id)
       }
 
-      for (let i=0; i<ids.length; i++ ){
+      for (let i = 0; i < ids.length; i++) {
         await createRow(ids[i])
       }
 
@@ -150,7 +160,7 @@ describe("/rows", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
@@ -196,8 +206,8 @@ describe("/rows", () => {
         .expect(200)
 
       expect(res.body.length).toBe(2)
-      expect(res.body.find(r => r.name === newRow.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find(r => r.name === row.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -215,92 +225,91 @@ describe("/rows", () => {
 
     it("row values are coerced", async () => {
       const str = {
-        type: "string",
+        type: FieldType.STRING,
+        name: "str",
         constraints: { type: "string", presence: false },
       }
       const attachment = {
-        type: "attachment",
+        type: FieldType.ATTACHMENT,
+        name: "attachment",
         constraints: { type: "array", presence: false },
       }
       const bool = {
-        type: "boolean",
+        type: FieldType.BOOLEAN,
+        name: "boolean",
         constraints: { type: "boolean", presence: false },
       }
       const number = {
-        type: "number",
+        type: FieldType.NUMBER,
+        name: "str",
         constraints: { type: "number", presence: false },
       }
       const datetime = {
-        type: "datetime",
+        type: FieldType.DATETIME,
+        name: "datetime",
         constraints: {
           type: "string",
           presence: false,
           datetime: { earliest: "", latest: "" },
-        }
+        },
       }
       const arrayField = {
-        type: "array",
+        type: FieldType.ARRAY,
         constraints: {
           type: "array",
           presence: false,
-          inclusion: [
+          inclusion: ["One", "Two", "Three"],
-            "One",
-            "Two",
-            "Three",
-          ]
         },
         name: "Sample Tags",
-        sortable: false
+        sortable: false,
       }
       const optsField = {
-        fieldName: "Sample Opts",
+          fieldName: "Sample Opts",
-        name: "Sample Opts",
+          name: "Sample Opts",
-        type: "options",
+          type: FieldType.OPTIONS,
-        constraints: {
+          constraints: {
-          type: "string",
+            type: "string",
-          presence: false,
+            presence: false,
-          inclusion: [ "Alpha", "Beta", "Gamma" ]
+            inclusion: ["Alpha", "Beta", "Gamma"],
+          },
         },
-      },
+        table = await config.createTable({
-
+          name: "TestTable2",
-      table = await config.createTable({
+          type: "table",
-        name: "TestTable2",
+          schema: {
-        type: "table",
+            name: str,
-        key: "name",
+            stringUndefined: str,
-        schema: {
+            stringNull: str,
-          name: str,
+            stringString: str,
-          stringUndefined: str,
+            numberEmptyString: number,
-          stringNull: str,
+            numberNull: number,
-          stringString: str,
+            numberUndefined: number,
-          numberEmptyString: number,
+            numberString: number,
-          numberNull: number,
+            numberNumber: number,
-          numberUndefined: number,
+            datetimeEmptyString: datetime,
-          numberString: number,
+            datetimeNull: datetime,
-          numberNumber: number,
+            datetimeUndefined: datetime,
-          datetimeEmptyString: datetime,
+            datetimeString: datetime,
-          datetimeNull: datetime,
+            datetimeDate: datetime,
-          datetimeUndefined: datetime,
+            boolNull: bool,
-          datetimeString: datetime,
+            boolEmpty: bool,
-          datetimeDate: datetime,
+            boolUndefined: bool,
-          boolNull: bool,
+            boolString: bool,
-          boolEmpty: bool,
+            boolBool: bool,
-          boolUndefined: bool,
+            attachmentNull: attachment,
-          boolString: bool,
+            attachmentUndefined: attachment,
-          boolBool: bool,
+            attachmentEmpty: attachment,
-          attachmentNull: attachment,
+            attachmentEmptyArrayStr: attachment,
-          attachmentUndefined: attachment,
+            arrayFieldEmptyArrayStr: arrayField,
-          attachmentEmpty: attachment,
+            arrayFieldArrayStrKnown: arrayField,
-          attachmentEmptyArrayStr: attachment,
+            arrayFieldNull: arrayField,
-          arrayFieldEmptyArrayStr: arrayField,
+            arrayFieldUndefined: arrayField,
-          arrayFieldArrayStrKnown: arrayField,
+            optsFieldEmptyStr: optsField,
-          arrayFieldNull: arrayField,
+            optsFieldUndefined: optsField,
-          arrayFieldUndefined: arrayField,
+            optsFieldNull: optsField,
-          optsFieldEmptyStr: optsField,
+            optsFieldStrKnown: optsField,
-          optsFieldUndefined: optsField,
+          },
-          optsFieldNull: optsField,
+        })
-          optsFieldStrKnown: optsField
-        },
-      })
 
       row = {
         name: "Test Row",
@@ -334,13 +343,13 @@ describe("/rows", () => {
         optsFieldEmptyStr: "",
         optsFieldUndefined: undefined,
         optsFieldNull: null,
-        optsFieldStrKnown: 'Alpha'
+        optsFieldStrKnown: "Alpha",
       }
 
-      const createdRow = await config.createRow(row);
+      const createdRow = await config.createRow(row)
-      const id = createdRow._id
+      const id = createdRow._id!
 
-      const saved = (await loadRow(id, table._id)).body
+      const saved = (await loadRow(id, table._id!)).body
 
       expect(saved.stringUndefined).toBe(undefined)
       expect(saved.stringNull).toBe("")
@@ -372,8 +381,8 @@ describe("/rows", () => {
       expect(saved.optsFieldEmptyStr).toEqual(null)
       expect(saved.optsFieldUndefined).toEqual(undefined)
       expect(saved.optsFieldNull).toEqual(null)
-      expect(saved.arrayFieldArrayStrKnown).toEqual(['One'])
+      expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-      expect(saved.optsFieldStrKnown).toEqual('Alpha')
+      expect(saved.optsFieldStrKnown).toEqual("Alpha")
     })
   })
 
@@ -396,13 +405,13 @@ describe("/rows", () => {
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual(
+      expect((res as any).res.statusMessage).toEqual(
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
       expect(res.body.description).toEqual(existing.description)
 
-      const savedRow = await loadRow(res.body._id, table._id)
+      const savedRow = await loadRow(res.body._id, table._id!)
 
       expect(savedRow.body.description).toEqual(existing.description)
       expect(savedRow.body.name).toEqual("Updated Name")
@@ -504,7 +513,7 @@ describe("/rows", () => {
         .expect(200)
 
       expect(res.body.length).toEqual(2)
-      await loadRow(row1._id, table._id, 404)
+      await loadRow(row1._id!, table._id!, 404)
       await assertRowUsage(rowUsage - 2)
       await assertQueryUsage(queryUsage + 1)
     })
@@ -562,7 +571,7 @@ describe("/rows", () => {
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
       const { table, firstRow, secondRow } = await tenancy.doInTenant(
-        setup.structures.TENANT_ID,
+        config.getTenantId(),
         async () => {
           const table = await config.createLinkedTable()
           const firstRow = await config.createRow({
@@ -624,7 +633,7 @@ describe("/rows", () => {
       await setup.switchToSelfHosted(async () => {
         context.doInAppContext(config.getAppId(), async () => {
           const enriched = await outputProcessing(table, [row])
-          expect(enriched[0].attachment[0].url).toBe(
+          expect((enriched as Row[])[0].attachment[0].url).toBe(
             `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
           )
         })

packages/server/src/automations/tests/queryRows.spec.ts

@@ -1,6 +1,6 @@
 // lucene searching not supported in test due to use of PouchDB
-let rows = []
+let rows: Row[] = []
-jest.mock("../../api/controllers/row/internalSearch", () => ({
+jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
   fullSearch: jest.fn(() => {
     return {
       rows,
@@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
   }),
   paginatedSearch: jest.fn(),
 }))
-const setup = require("./utilities")
+import { Row, Table } from "@budibase/types"
+import * as setup from "./utilities"
 
 const NAME = "Test"
 
 describe("Test a query step automation", () => {
-  let table
+  let table: Table
   let config = setup.getConfig()
 
   beforeAll(async () => {
@@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
       filters: {},
       "filters-def": [
         {
-          value: null
+          value: null,
-        }
+        },
       ],
       sortColumn: "name",
       sortOrder: "ascending",

packages/server/src/integrations/microsoftSqlServer.ts

@@ -233,8 +233,11 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     try {
       // if encrypt is undefined, then default is to encrypt
       const encrypt = this.config.encrypt === undefined || this.config.encrypt
-      const clientCfg: MSSQLConfig & sqlServer.config = {
+      const clientCfg: sqlServer.config = {
-        ...this.config,
+        user: this.config.user,
+        password: this.config.password,
+        server: this.config.server,
+        database: this.config.database,
         port: +this.config.port,
         options: {
           encrypt,
@@ -244,11 +247,11 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       if (encrypt) {
         clientCfg.options!.trustServerCertificate = true
       }
-      delete clientCfg.encrypt
 
       switch (this.config.authType) {
         case MSSQLConfigAuthType.AZURE_ACTIVE_DIRECTORY:
-          const { clientId, tenantId, clientSecret } = this.config.adConfig
+          const { clientId, tenantId, clientSecret } =
+            this.config.adConfig || {}
           const clientApp = new ConfidentialClientApplication({
             auth: {
               clientId,
@@ -269,7 +272,8 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
           }
           break
         case MSSQLConfigAuthType.NTLM:
-          const { domain, trustServerCertificate } = this.config.ntlmConfig
+          const { domain, trustServerCertificate } =
+            this.config.ntlmConfig || {}
           clientCfg.authentication = {
             type: "ntlm",
             options: {
@@ -277,8 +281,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
             },
           }
           clientCfg.options ??= {}
-          clientCfg.options.trustServerCertificate = trustServerCertificate
+          clientCfg.options.trustServerCertificate = !!trustServerCertificate
           break
+        case null:
         case undefined:
           break
         default:

packages/server/src/sdk/app/rows/index.ts

@@ -1,7 +1,11 @@
 import * as attachments from "./attachments"
 import * as rows from "./rows"
+import * as search from "./search"
+import * as utils from "./utils"
 
 export default {
   ...attachments,
   ...rows,
+  ...search,
+  utils: utils,
 }

packages/server/src/sdk/app/rows/search.ts

@@ -0,0 +1,66 @@
+import { SearchFilters } from "@budibase/types"
+import { isExternalTable } from "../../../integrations/utils"
+import * as internal from "./search/internal"
+import * as external from "./search/external"
+import { Format } from "../../../api/controllers/view/exporters"
+
+export interface SearchParams {
+  tableId: string
+  paginate?: boolean
+  query: SearchFilters
+  bookmark?: string
+  limit?: number
+  sort?: string
+  sortOrder?: string
+  sortType?: string
+  version?: string
+  disableEscaping?: boolean
+}
+
+export interface ViewParams {
+  calculation: string
+  group: string
+  field: string
+}
+
+function pickApi(tableId: any) {
+  if (isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+export async function search(options: SearchParams) {
+  return pickApi(options.tableId).search(options)
+}
+
+export interface ExportRowsParams {
+  tableId: string
+  format: Format
+  rowIds?: string[]
+  columns?: string[]
+  query: SearchFilters
+}
+
+export interface ExportRowsResult {
+  fileName: string
+  content: string
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  return pickApi(options.tableId).exportRows(options)
+}
+
+export async function fetch(tableId: string) {
+  return pickApi(tableId).fetch(tableId)
+}
+
+export async function fetchView(
+  tableId: string,
+  viewName: string,
+  params: ViewParams
+) {
+  return pickApi(tableId).fetchView(viewName, params)
+}

packages/server/src/sdk/app/rows/search/external.ts

@@ -0,0 +1,172 @@
+import {
+  SortJson,
+  SortDirection,
+  Operation,
+  PaginationJson,
+  IncludeRelationship,
+  Row,
+  SearchFilters,
+} from "@budibase/types"
+import * as exporters from "../../../../api/controllers/view/exporters"
+import sdk from "../../../../sdk"
+import { handleRequest } from "../../../../api/controllers/row/external"
+import { breakExternalTableId } from "../../../../integrations/utils"
+import { cleanExportRows } from "../utils"
+import { utils } from "@budibase/shared-core"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { HTTPError } from "@budibase/backend-core"
+
+export async function search(options: SearchParams) {
+  const { tableId } = options
+  const { paginate, query, ...params } = options
+  const { limit } = params
+  let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
+  if (paginate && !bookmark) {
+    bookmark = 1
+  }
+  let paginateObj = {}
+
+  if (paginate) {
+    paginateObj = {
+      // add one so we can track if there is another page
+      limit: limit,
+      page: bookmark,
+    }
+  } else if (params && limit) {
+    paginateObj = {
+      limit: limit,
+    }
+  }
+  let sort: SortJson | undefined
+  if (params.sort) {
+    const direction =
+      params.sortOrder === "descending"
+        ? SortDirection.DESCENDING
+        : SortDirection.ASCENDING
+    sort = {
+      [params.sort]: { direction },
+    }
+  }
+  try {
+    const rows = (await handleRequest(Operation.READ, tableId, {
+      filters: query,
+      sort,
+      paginate: paginateObj as PaginationJson,
+      includeSqlRelationships: IncludeRelationship.INCLUDE,
+    })) as Row[]
+    let hasNextPage = false
+    if (paginate && rows.length === limit) {
+      const nextRows = (await handleRequest(Operation.READ, tableId, {
+        filters: query,
+        sort,
+        paginate: {
+          limit: 1,
+          page: bookmark! * limit + 1,
+        },
+        includeSqlRelationships: IncludeRelationship.INCLUDE,
+      })) as Row[]
+      hasNextPage = nextRows.length > 0
+    }
+    // need wrapper object for bookmarks etc when paginating
+    return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
+  } catch (err: any) {
+    if (err.message && err.message.includes("does not exist")) {
+      throw new Error(
+        `Table updated externally, please re-fetch - ${err.message}`
+      )
+    } else {
+      throw err
+    }
+  }
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, columns, rowIds } = options
+  const { datasourceId, tableName } = breakExternalTableId(tableId)
+
+  let query: SearchFilters = {}
+  if (rowIds?.length) {
+    query = {
+      oneOf: {
+        _id: rowIds.map((row: string) => {
+          const ids = JSON.parse(
+            decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
+          )
+          if (ids.length > 1) {
+            throw new HTTPError(
+              "Export data does not support composite keys.",
+              400
+            )
+          }
+          return ids[0]
+        }),
+      },
+    }
+  }
+
+  const datasource = await sdk.datasources.get(datasourceId!)
+  if (!datasource || !datasource.entities) {
+    throw new HTTPError("Datasource has not been configured for plus API.", 400)
+  }
+
+  let result = await search({ tableId, query })
+  let rows: Row[] = []
+
+  // Filter data to only specified columns if required
+
+  if (columns && columns.length) {
+    for (let i = 0; i < result.rows.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result.rows[i][column]
+      }
+    }
+  } else {
+    rows = result.rows
+  }
+
+  if (!tableName) {
+    throw new HTTPError("Could not find table name.", 400)
+  }
+  const schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
+  let content: string
+  switch (format) {
+    case exporters.Format.CSV:
+      content = exporters.csv(headers, exportRows)
+      break
+    case exporters.Format.JSON:
+      content = exporters.json(exportRows)
+      break
+    case exporters.Format.JSON_WITH_SCHEMA:
+      content = exporters.jsonWithSchema(schema, exportRows)
+      break
+    default:
+      throw utils.unreachable(format)
+  }
+
+  const fileName = `export.${format}`
+  return {
+    fileName,
+    content,
+  }
+}
+
+export async function fetch(tableId: string) {
+  return handleRequest(Operation.READ, tableId, {
+    includeSqlRelationships: IncludeRelationship.INCLUDE,
+  })
+}
+
+export async function fetchView(viewName: string) {
+  // there are no views in external datasources, shouldn't ever be called
+  // for now just fetch
+  const split = viewName.split("all_")
+  const tableId = split[1] ? split[1] : split[0]
+  return fetch(tableId)
+}

packages/server/src/sdk/app/rows/search/internal.ts

@@ -0,0 +1,261 @@
+import {
+  context,
+  SearchParams as InternalSearchParams,
+} from "@budibase/backend-core"
+import env from "../../../../environment"
+import { fullSearch, paginatedSearch } from "./internalSearch"
+import {
+  InternalTables,
+  getRowParams,
+  DocumentType,
+} from "../../../../db/utils"
+import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
+import { Database, Row } from "@budibase/types"
+import { cleanExportRows } from "../utils"
+import {
+  Format,
+  csv,
+  json,
+  jsonWithSchema,
+} from "../../../../api/controllers/view/exporters"
+import * as inMemoryViews from "../../../../db/inMemoryView"
+import {
+  migrateToInMemoryView,
+  migrateToDesignView,
+  getFromDesignDoc,
+  getFromMemoryDoc,
+} from "../../../../api/controllers/view/utils"
+import sdk from "../../../../sdk"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+
+export async function search(options: SearchParams) {
+  const { tableId } = options
+
+  // Fetch the whole table when running in cypress, as search doesn't work
+  if (!env.COUCH_DB_URL && env.isCypress()) {
+    return { rows: await fetch(tableId) }
+  }
+
+  const db = context.getAppDB()
+  const { paginate, query } = options
+
+  const params: InternalSearchParams<any> = {
+    tableId: options.tableId,
+    sort: options.sort,
+    sortOrder: options.sortOrder,
+    sortType: options.sortType,
+    limit: options.limit,
+    bookmark: options.bookmark,
+    version: options.version,
+    disableEscaping: options.disableEscaping,
+  }
+
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type === "number" ? "number" : "string"
+  }
+
+  let response
+  if (paginate) {
+    response = await paginatedSearch(query, params)
+  } else {
+    response = await fullSearch(query, params)
+  }
+
+  // Enrich search results with relationships
+  if (response.rows && response.rows.length) {
+    // enrich with global users if from users table
+    if (tableId === InternalTables.USER_METADATA) {
+      response.rows = await getGlobalUsersFromMetadata(response.rows)
+    }
+    table = table || (await db.get(tableId))
+    response.rows = await outputProcessing(table, response.rows)
+  }
+
+  return response
+}
+
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, rowIds, columns, query } = options
+  const db = context.getAppDB()
+  const table = await db.get(tableId)
+
+  let result
+  if (rowIds) {
+    let response = (
+      await db.allDocs({
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await search({ tableId, query })
+    result = searchResponse.rows
+  }
+
+  let rows: Row[] = []
+  let schema = table.schema
+
+  // Filter data to only specified columns if required
+  if (columns && columns.length) {
+    for (let i = 0; i < result.length; i++) {
+      rows[i] = {}
+      for (let column of columns) {
+        rows[i][column] = result[i][column]
+      }
+    }
+  } else {
+    rows = result
+  }
+
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+  if (format === Format.CSV) {
+    return {
+      fileName: "export.csv",
+      content: csv(Object.keys(rows[0]), exportRows),
+    }
+  } else if (format === Format.JSON) {
+    return {
+      fileName: "export.json",
+      content: json(exportRows),
+    }
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    return {
+      fileName: "export.json",
+      content: jsonWithSchema(schema, exportRows),
+    }
+  } else {
+    throw "Format not recognised"
+  }
+}
+
+export async function fetch(tableId: string) {
+  const db = context.getAppDB()
+
+  let table = await db.get(tableId)
+  let rows = await getRawTableData(db, tableId)
+  const result = await outputProcessing(table, rows)
+  return result
+}
+
+async function getRawTableData(db: Database, tableId: string) {
+  let rows
+  if (tableId === InternalTables.USER_METADATA) {
+    rows = await sdk.users.fetchMetadata()
+  } else {
+    const response = await db.allDocs(
+      getRowParams(tableId, null, {
+        include_docs: true,
+      })
+    )
+    rows = response.rows.map(row => row.doc)
+  }
+  return rows as Row[]
+}
+
+export async function fetchView(
+  viewName: string,
+  options: { calculation: string; group: string; field: string }
+) {
+  // if this is a table view being looked for just transfer to that
+  if (viewName.startsWith(DocumentType.TABLE)) {
+    return fetch(viewName)
+  }
+
+  const db = context.getAppDB()
+  const { calculation, group, field } = options
+  const viewInfo = await getView(db, viewName)
+  let response
+  if (env.SELF_HOSTED) {
+    response = await db.query(`database/${viewName}`, {
+      include_docs: !calculation,
+      group: !!group,
+    })
+  } else {
+    const tableId = viewInfo.meta.tableId
+    const data = await getRawTableData(db, tableId)
+    response = await inMemoryViews.runView(
+      viewInfo,
+      calculation as string,
+      !!group,
+      data
+    )
+  }
+
+  let rows
+  if (!calculation) {
+    response.rows = response.rows.map(row => row.doc)
+    let table
+    try {
+      table = await db.get(viewInfo.meta.tableId)
+    } catch (err) {
+      /* istanbul ignore next */
+      table = {
+        schema: {},
+      }
+    }
+    rows = await outputProcessing(table, response.rows)
+  }
+
+  if (calculation === CALCULATION_TYPES.STATS) {
+    response.rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      ...row.value,
+      avg: row.value.sum / row.value.count,
+    }))
+    rows = response.rows
+  }
+
+  if (
+    calculation === CALCULATION_TYPES.COUNT ||
+    calculation === CALCULATION_TYPES.SUM
+  ) {
+    rows = response.rows.map(row => ({
+      group: row.key,
+      field,
+      value: row.value,
+    }))
+  }
+  return rows
+}
+
+const CALCULATION_TYPES = {
+  SUM: "sum",
+  COUNT: "count",
+  STATS: "stats",
+}
+
+async function getView(db: Database, viewName: string) {
+  let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
+  let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
+  let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
+  let viewInfo,
+    migrate = false
+  try {
+    viewInfo = await mainGetter(db, viewName)
+  } catch (err: any) {
+    // check if it can be retrieved from design doc (needs migrated)
+    if (err.status !== 404) {
+      viewInfo = null
+    } else {
+      viewInfo = await secondaryGetter(db, viewName)
+      migrate = !!viewInfo
+    }
+  }
+  if (migrate) {
+    await migration(db, viewName)
+  }
+  if (!viewInfo) {
+    throw "View does not exist."
+  }
+  return viewInfo
+}

packages/server/src/sdk/app/rows/utils.ts

@@ -0,0 +1,48 @@
+import { TableSchema } from "@budibase/types"
+import { FieldTypes } from "../../../constants"
+import { makeExternalQuery } from "../../../integrations/base/query"
+import { Format } from "../../../api/controllers/view/exporters"
+import sdk from "../.."
+
+export async function getDatasourceAndQuery(json: any) {
+  const datasourceId = json.endpoint.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId)
+  return makeExternalQuery(datasource, json)
+}
+
+export function cleanExportRows(
+  rows: any[],
+  schema: TableSchema,
+  format: string,
+  columns?: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  if (format === Format.CSV) {
+    // Intended to append empty values in export
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = undefined
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -0,0 +1,101 @@
+import { exportRows } from "../../app/rows/search/external"
+import sdk from "../.."
+import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
+import { ExportRowsParams } from "../../app/rows/search"
+import { Format } from "../../../api/controllers/view/exporters"
+import { HTTPError } from "@budibase/backend-core"
+import { Operation } from "@budibase/types"
+
+const mockDatasourcesGet = jest.fn()
+sdk.datasources.get = mockDatasourcesGet
+
+jest.mock("../../../api/controllers/row/ExternalRequest")
+
+jest.mock("../../../api/controllers/view/exporters", () => ({
+  ...jest.requireActual("../../../api/controllers/view/exporters"),
+  csv: jest.fn(),
+  Format: {
+    CSV: "csv",
+  },
+}))
+jest.mock("../../../utilities/fileSystem")
+
+describe("external row sdk", () => {
+  describe("exportRows", () => {
+    function getExportOptions(): ExportRowsParams {
+      return {
+        tableId: "datasource__tablename",
+        format: Format.CSV,
+        query: {},
+      }
+    }
+
+    const externalRequestCall = jest.fn()
+    beforeAll(() => {
+      jest
+        .spyOn(ExternalRequest.prototype, "run")
+        .mockImplementation(externalRequestCall.mockResolvedValue([]))
+    })
+
+    afterEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("should throw a 400 if no datasource entities are present", async () => {
+      const exportOptions = getExportOptions()
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Datasource has not been configured for plus API.", 400)
+      )
+    })
+
+    it("should handle single quotes from a row ID", async () => {
+      mockDatasourcesGet.mockImplementation(async () => ({
+        entities: {
+          tablename: {
+            schema: {},
+          },
+        },
+      }))
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["['d001']"]
+
+      await exportRows(exportOptions)
+
+      expect(ExternalRequest).toBeCalledTimes(1)
+      expect(ExternalRequest).toBeCalledWith(
+        Operation.READ,
+        exportOptions.tableId,
+        undefined
+      )
+
+      expect(externalRequestCall).toBeCalledTimes(1)
+      expect(externalRequestCall).toBeCalledWith(
+        expect.objectContaining({
+          filters: {
+            oneOf: {
+              _id: ["d001"],
+            },
+          },
+        })
+      )
+    })
+
+    it("should throw a 400 if any composite keys are present", async () => {
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"]
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Export data does not support composite keys.", 400)
+      )
+    })
+
+    it("should throw a 400 if no table name was found", async () => {
+      const exportOptions = getExportOptions()
+      exportOptions.tableId = "datasource__"
+      exportOptions.rowIds = ["[123]"]
+
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Could not find table name.", 400)
+      )
+    })
+  })
+})

packages/server/src/sdk/users/index.ts

@@ -1,5 +1,7 @@
 import * as utils from "./utils"
+import * as sessions from "./sessions"
 
 export default {
   ...utils,
+  sessions,
 }

packages/server/src/sdk/users/sessions.ts

@@ -0,0 +1,38 @@
+import { builderSocket } from "../../websockets"
+import { App, SocketSession } from "@budibase/types"
+
+export const enrichApps = async (apps: App[]) => {
+  // Sessions can only exist for dev app IDs
+  const devAppIds = apps
+    .filter((app: any) => app.status === "development")
+    .map((app: any) => app.appId)
+
+  // Get all sessions for all apps and enrich app list
+  const sessions = await builderSocket?.getRoomSessions(devAppIds)
+  if (sessions?.length) {
+    let appSessionMap: Record<string, SocketSession[]> = {}
+    sessions.forEach(session => {
+      const room = session.room
+      if (!room) {
+        return
+      }
+      if (!appSessionMap[room]) {
+        appSessionMap[room] = []
+      }
+      appSessionMap[room].push(session)
+    })
+    return apps.map(app => {
+      // Shallow clone to avoid mutating original reference
+      let enriched = { ...app }
+      const sessions = appSessionMap[app.appId]
+      if (sessions?.length) {
+        enriched.sessions = sessions
+      } else {
+        delete enriched.sessions
+      }
+      return enriched
+    })
+  } else {
+    return apps
+  }
+}

packages/server/src/sdk/users/utils.ts

@@ -64,6 +64,25 @@ export async function rawUserMetadata(db?: Database) {
   ).rows.map(row => row.doc)
 }
 
+export async function fetchMetadata() {
+  const global = await getGlobalUsers()
+  const metadata = await rawUserMetadata()
+  const users = []
+  for (let user of global) {
+    // find the metadata that matches up to the global ID
+    const info = metadata.find(meta => meta._id.includes(user._id))
+    // remove these props, not for the correct DB
+    users.push({
+      ...user,
+      ...info,
+      tableId: InternalTables.USER_METADATA,
+      // make sure the ID is always a local ID, not a global one
+      _id: generateUserMetadataID(user._id),
+    })
+  }
+  return users
+}
+
 export async function syncGlobalUsers() {
   // sync user metadata
   const dbs = [context.getDevAppDB(), context.getProdAppDB()]

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -135,7 +135,10 @@ class TestConfiguration {
     }
   }
 
-  async doInContext(appId: string | null, task: any) {
+  async doInContext<T>(
+    appId: string | null,
+    task: () => Promise<T>
+  ): Promise<T> {
     if (!appId) {
       appId = this.appId
     }

packages/server/src/websockets/websocket.ts

@@ -125,9 +125,18 @@ export class BaseSocket {
   }
 
   // Gets an array of all redis keys of users inside a certain room
-  async getRoomSessionIds(room: string): Promise<string[]> {
+  async getRoomSessionIds(room: string | string[]): Promise<string[]> {
-    const keys = await this.redisClient?.get(this.getRoomKey(room))
+    if (Array.isArray(room)) {
-    return keys || []
+      const roomKeys = room.map(this.getRoomKey.bind(this))
+      const roomSessionIdMap = await this.redisClient?.bulkGet(roomKeys)
+      let sessionIds: any[] = []
+      Object.values(roomSessionIdMap || {}).forEach(roomSessionIds => {
+        sessionIds = sessionIds.concat(roomSessionIds)
+      })
+      return sessionIds
+    } else {
+      return (await this.redisClient?.get(this.getRoomKey(room))) || []
+    }
   }
 
   // Sets the list of redis keys for users inside a certain room.
@@ -137,7 +146,7 @@ export class BaseSocket {
   }
 
   // Gets a list of all users inside a certain room
-  async getRoomSessions(room?: string): Promise<SocketSession[]> {
+  async getRoomSessions(room?: string | string[]): Promise<SocketSession[]> {
     if (room) {
       const sessionIds = await this.getRoomSessionIds(room)
       const keys = sessionIds.map(this.getSessionKey.bind(this))

packages/types/src/documents/app/app.ts

@@ -1,4 +1,5 @@
 import { User, Document } from "../"
+import { SocketSession } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
 
@@ -17,6 +18,7 @@ export interface App extends Document {
   customTheme?: AppCustomTheme
   revertableVersion?: string
   lockedBy?: User
+  sessions?: SocketSession[]
   navigation?: AppNavigation
   automationErrors?: AppMetadataErrors
   icon?: AppIcon

packages/types/src/sdk/search.ts

@@ -32,7 +32,7 @@ export interface SearchFilters {
     [key: string]: any[]
   }
   contains?: {
-    [key: string]: any[]
+    [key: string]: any[] | any
   }
   notContains?: {
     [key: string]: any[]