Merge master.

hosting/.env

@@ -17,6 +17,7 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
 WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
@@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
 
 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
 PLUGINS_DIR=
-ROLLING_LOG_MAX_SIZE=
+ROLLING_LOG_MAX_SIZE=

packages/backend-core/src/environment.ts

@@ -107,7 +107,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/client/src/components/app/NavItem.svelte

@@ -157,6 +157,11 @@
     width: 100%;
   }
 
+  /* Use normal theme colors for links when using a top nav */
+  .dropdown:not(.left) .sublinks a {
+    color: var(--spectrum-alias-text-color);
+  }
+
   /* Left dropdowns */
   .dropdown.left .sublinks-wrapper {
     display: none;

packages/server/package.json

@@ -125,7 +125,7 @@
     "@babel/preset-env": "7.16.11",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
-    "@types/archiver": "^6.0.2",
+    "@types/archiver": "6.0.2",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "29.5.5",

packages/server/src/api/controllers/application.ts

@@ -320,6 +320,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
         "theme",
         "customTheme",
         "icon",
+        "snippets",
       ]
       keys.forEach(key => {
         if (existing[key]) {

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { db as dbCore } from "@budibase/backend-core"
-import AliasTables from "./alias"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 
@@ -618,7 +617,7 @@ export class ExternalRequest<T extends Operation> {
     if (env.SQL_ALIASING_DISABLE) {
       response = await getDatasourceAndQuery(json)
     } else {
-      const aliasing = new AliasTables(Object.keys(this.tables))
+      const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
       response = await aliasing.queryWithAliasing(json)
     }
 

packages/server/src/api/controllers/row/utils/basic.ts

@@ -62,12 +62,12 @@ export function basicProcessing({
   row,
   table,
   isLinked,
-  internal,
+  sqs,
 }: {
   row: Row
   table: Table
   isLinked: boolean
-  internal?: boolean
+  sqs?: boolean
 }): Row {
   const thisRow: Row = {}
   // filter the row down to what is actually the row (not joined)
@@ -84,12 +84,13 @@ export function basicProcessing({
       thisRow[fieldName] = value
     }
   }
-  if (!internal) {
+  if (!sqs) {
     thisRow._id = generateIdForRow(row, table, isLinked)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
   } else {
-    for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+    const columns = Object.keys(table.schema)
+    for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
       thisRow[internalColumn] = extractFieldValue({
         row,
         tableName: table._id!,

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
       continue
     }
 
-    let linked = await basicProcessing({
+    let linked = basicProcessing({
       row,
       table: linkedTable,
       isLinked: true,
-      internal: opts?.sqs,
+      sqs: opts?.sqs,
     })
     if (!linked._id) {
       continue

packages/server/src/api/controllers/row/utils/utils.ts

@@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
     let rowId = row._id
     if (opts?.sqs) {
       rowId = getInternalRowId(row, table)
+      row._id = rowId
     } else if (!rowId) {
       rowId = generateIdForRow(row, table)
       row._id = rowId
@@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
         row,
         table,
         isLinked: false,
-        internal: opts?.sqs,
+        sqs: opts?.sqs,
       }),
       table
     )
@@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
       tables,
       row,
       finalRows,
-      relationships
+      relationships,
+      opts
     )
   }
 

packages/server/src/db/utils.ts

@@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
 export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
 export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
+export const SQS_DATASOURCE_INTERNAL = "internal"
 export const ViewName = dbCore.ViewName
 export const InternalTables = dbCore.InternalTable
 export const UNICODE_MAX = dbCore.UNICODE_MAX

packages/server/src/environment.ts

@@ -28,6 +28,7 @@ const DEFAULTS = {
   PLUGINS_DIR: "/plugins",
   FORKED_PROCESS_NAME: "main",
   JS_RUNNER_MEMORY_LIMIT: 64,
+  COUCH_DB_SQL_URL: "http://localhost:4006",
 }
 
 const QUERY_THREAD_TIMEOUT =
@@ -39,6 +40,7 @@ const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
   AWS_REGION: process.env.AWS_REGION,

packages/server/src/integrations/base/sql.ts

@@ -22,6 +22,8 @@ import {
   SortDirection,
   SqlQueryBinding,
   Table,
+  TableSourceType,
+  INTERNAL_TABLE_SOURCE_ID,
 } from "@budibase/types"
 import environment from "../../environment"
 
@@ -135,6 +137,18 @@ function generateSelectStatement(
   })
 }
 
+function getTableName(table?: Table): string | undefined {
+  // SQS uses the table ID rather than the table name
+  if (
+    table?.sourceType === TableSourceType.INTERNAL ||
+    table?.sourceId === INTERNAL_TABLE_SOURCE_ID
+  ) {
+    return table?._id
+  } else {
+    return table?.name
+  }
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -149,7 +163,7 @@ class InternalBuilder {
     table: Table,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): Knex.QueryBuilder {
-    function getTableName(name: string) {
+    function getTableAlias(name: string) {
       const alias = opts.aliases?.[name]
       return alias || name
     }
@@ -161,11 +175,11 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${getTableName(table.name)}.${updatedKey}`, value)
+          fn(`${getTableName(table)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
           const [filterTableName, property] = updatedKey.split(".")
-          fn(`${getTableName(filterTableName)}.${property}`, value)
+          fn(`${getTableAlias(filterTableName)}.${property}`, value)
         }
       }
     }
@@ -379,9 +393,10 @@ class InternalBuilder {
   addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
     let { sort, paginate } = json
     const table = json.meta?.table
+    const tableName = getTableName(table)
     const aliases = json.tableAliases
     const aliased =
-      table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
+      tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
     if (sort && Object.keys(sort || {}).length > 0) {
       for (let [key, value] of Object.entries(sort)) {
         const direction =
@@ -762,12 +777,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     results: Record<string, any>[],
     aliases?: Record<string, string>
   ): Record<string, any>[] {
+    const tableName = getTableName(table)
     for (const [name, field] of Object.entries(table.schema)) {
       if (!this._isJsonColumn(field)) {
         continue
       }
-      const tableName = aliases?.[table.name] || table.name
+      const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
-      const fullName = `${tableName}.${name}`
+      const fullName = `${aliasedTableName}.${name}`
       for (let row of results) {
         if (typeof row[fullName] === "string") {
           row[fullName] = JSON.parse(row[fullName])

packages/server/src/integrations/base/sqlTable.ts

@@ -1,19 +1,20 @@
 import { Knex, knex } from "knex"
 import {
-  RelationshipType,
   FieldSubtype,
+  FieldType,
   NumberFieldMetadata,
   Operation,
   QueryJson,
+  RelationshipType,
   RenameColumn,
-  Table,
-  FieldType,
   SqlQuery,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
 import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
+import { utils } from "@budibase/shared-core"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-import { utils } from "@budibase/shared-core"
 
 function isIgnoredType(type: FieldType) {
   const ignored = [FieldType.LINK, FieldType.FORMULA]
@@ -105,13 +106,13 @@ function generateSchema(
           column.relationshipType !== RelationshipType.MANY_TO_MANY
         ) {
           if (!column.foreignKey || !column.tableId) {
-            throw "Invalid relationship schema"
+            throw new Error("Invalid relationship schema")
           }
           const { tableName } = breakExternalTableId(column.tableId)
           // @ts-ignore
           const relatedTable = tables[tableName]
           if (!relatedTable) {
-            throw "Referenced table doesn't exist"
+            throw new Error("Referenced table doesn't exist")
           }
           const relatedPrimary = relatedTable.primary[0]
           const externalType = relatedTable.schema[relatedPrimary].externalType
@@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
 
     let query: Knex.SchemaBuilder
     if (!json.table || !json.meta || !json.meta.tables) {
-      throw "Cannot execute without table being specified"
+      throw new Error("Cannot execute without table being specified")
     }
+    if (json.table.sourceType === TableSourceType.INTERNAL) {
+      throw new Error("Cannot perform table actions for SQS.")
+    }
+
     switch (this._operation(json)) {
       case Operation.CREATE_TABLE:
         query = buildCreateTable(client, json.table, json.meta.tables)
         break
       case Operation.UPDATE_TABLE:
         if (!json.meta || !json.meta.table) {
-          throw "Must specify old table for update"
+          throw new Error("Must specify old table for update")
         }
         // renameColumn does not work for MySQL, so return a raw query
         if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
         query = buildDeleteTable(client, json.table)
         break
       default:
-        throw "Table operation is of unknown type"
+        throw new Error("Table operation is of unknown type")
     }
     return getNativeSql(query)
   }

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -8,8 +8,10 @@ import {
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
-import AliasTables from "../../api/controllers/row/alias"
 import { generator } from "@budibase/backend-core/tests"
+import sdk from "../../sdk"
+
+const AliasTables = sdk.rows.AliasTables
 
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")

packages/server/src/sdk/app/rows/index.ts

@@ -3,6 +3,7 @@ import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
 import * as external from "./external"
+import AliasTables from "./sqlAlias"
 
 export default {
   ...attachments,
@@ -10,4 +11,5 @@ export default {
   ...search,
   utils,
   external,
+  AliasTables,
 }

packages/server/src/sdk/app/rows/search.ts

@@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
 import env from "../../../environment"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
+import sdk from "../../index"
+import { searchInputMapping } from "./search/utils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -76,12 +78,15 @@ export async function search(
     }
   }
 
+  const table = await sdk.tables.getTable(options.tableId)
+  options = searchInputMapping(table, options)
+
   if (isExternalTable) {
-    return external.search(options)
+    return external.search(options, table)
   } else if (env.SQS_SEARCH_ENABLE) {
-    return sqs.search(options)
+    return sqs.search(options, table)
   } else {
-    return internal.search(options)
+    return internal.search(options, table)
   }
 }
 

packages/server/src/sdk/app/rows/search/external.ts

@@ -8,6 +8,7 @@ import {
   SearchFilters,
   RowSearchParams,
   SearchResponse,
+  Table,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import { handleRequest } from "../../../../api/controllers/row/external"
@@ -18,13 +19,13 @@ import {
 import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "./types"
 import { HTTPError, db } from "@budibase/backend-core"
-import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import sdk from "../../../"
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
   const { tableId } = options
   const { paginate, query, ...params } = options
@@ -68,8 +69,6 @@ export async function search(
   }
 
   try {
-    const table = await sdk.tables.getTable(tableId)
-    options = searchInputMapping(table, options)
     let rows = await handleRequest(Operation.READ, tableId, {
       filters: query,
       sort,
@@ -150,11 +149,15 @@ export async function exportRows(
   }
 
   const datasource = await sdk.datasources.get(datasourceId!)
+  const table = await sdk.tables.getTable(tableId)
   if (!datasource || !datasource.entities) {
     throw new HTTPError("Datasource has not been configured for plus API.", 400)
   }
 
-  let result = await search({ tableId, query: requestQuery, sort, sortOrder })
+  let result = await search(
+    { tableId, query: requestQuery, sort, sortOrder },
+    table
+  )
   let rows: Row[] = []
   let headers
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,6 +1,6 @@
 import { context, db, HTTPError } from "@budibase/backend-core"
 import env from "../../../../environment"
-import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
+import { fullSearch, paginatedSearch } from "./utils"
 import { getRowParams, InternalTables } from "../../../../db/utils"
 import {
   Database,
@@ -33,7 +33,8 @@ import pick from "lodash/pick"
 import { breakRowIdField } from "../../../../integrations/utils"
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
   const { tableId } = options
 
@@ -51,8 +52,6 @@ export async function search(
     query: {},
   }
 
-  let table = await sdk.tables.getTable(tableId)
-  options = searchInputMapping(table, options)
   if (params.sort && !params.sortType) {
     const schema = table.schema
     const sortField = schema[params.sort]
@@ -122,12 +121,15 @@ export async function exportRows(
 
     result = await outputProcessing<Row[]>(table, response)
   } else if (query) {
-    let searchResponse = await search({
+    let searchResponse = await search(
-      tableId,
+      {
-      query,
+        tableId,
-      sort,
+        query,
-      sortOrder,
+        sort,
-    })
+        sortOrder,
+      },
+      table
+    )
     result = searchResponse.rows
   }
 

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -20,7 +20,12 @@ import {
 } from "../../../../api/controllers/row/utils"
 import sdk from "../../../index"
 import { context } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  SQS_DATASOURCE_INTERNAL,
+} from "../../../../db/utils"
+import AliasTables from "../sqlAlias"
+import { outputProcessing } from "../../../../utilities/rowProcessor"
 
 function buildInternalFieldList(
   table: Table,
@@ -31,19 +36,19 @@ function buildInternalFieldList(
   fieldList = fieldList.concat(
     CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
   )
-  if (opts.relationships) {
+  for (let col of Object.values(table.schema)) {
-    for (let col of Object.values(table.schema)) {
+    const isRelationship = col.type === FieldType.LINK
-      if (col.type === FieldType.LINK) {
+    if (!opts.relationships && isRelationship) {
-        const linkCol = col as RelationshipFieldMetadata
+      continue
-        const relatedTable = tables.find(
+    }
-          table => table._id === linkCol.tableId
+    if (isRelationship) {
-        )!
+      const linkCol = col as RelationshipFieldMetadata
-        fieldList = fieldList.concat(
+      const relatedTable = tables.find(table => table._id === linkCol.tableId)!
-          buildInternalFieldList(relatedTable, tables, { relationships: false })
+      fieldList = fieldList.concat(
-        )
+        buildInternalFieldList(relatedTable, tables, { relationships: false })
-      } else {
+      )
-        fieldList.push(`${table._id}.${col.name}`)
+    } else {
-      }
+      fieldList.push(`${table._id}.${col.name}`)
     }
   }
   return fieldList
@@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
 }
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  table: Table
 ): Promise<SearchResponse<Row>> {
-  const { tableId, paginate, query, ...params } = options
+  const { paginate, query, ...params } = options
 
   const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
-  const table = allTables.find(table => table._id === tableId)
   if (!table) {
     throw new Error("Unable to find table")
   }
@@ -111,7 +116,7 @@ export async function search(
   const request: QueryJson = {
     endpoint: {
       // not important, we query ourselves
-      datasourceId: "internal",
+      datasourceId: SQS_DATASOURCE_INTERNAL,
       entityId: table._id!,
       operation: Operation.READ,
     },
@@ -154,34 +159,44 @@ export async function search(
     }
   }
   try {
-    const query = builder._query(request, {
+    const alias = new AliasTables(allTables.map(table => table.name))
-      disableReturning: true,
+    const rows = await alias.queryWithAliasing(request, async json => {
+      const query = builder._query(json, {
+        disableReturning: true,
+      })
+
+      if (Array.isArray(query)) {
+        throw new Error("SQS cannot currently handle multiple queries")
+      }
+
+      let sql = query.sql,
+        bindings = query.bindings
+
+      // quick hack for docIds
+      sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+      sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+      const db = context.getAppDB()
+      return await db.sql<Row>(sql, bindings)
     })
 
-    if (Array.isArray(query)) {
+    // process from the format of tableId.column to expected format
-      throw new Error("SQS cannot currently handle multiple queries")
+    const processed = await sqlOutputProcessing(
-    }
+      rows,
-
+      table!,
-    let sql = query.sql,
+      allTablesMap,
-      bindings = query.bindings
+      relationships,
-
+      {
-    // quick hack for docIds
+        sqs: true,
-    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+      }
-    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+    )
-
-    const db = context.getAppDB()
-    const rows = await db.sql<Row>(sql, bindings)
 
     return {
-      rows: await sqlOutputProcessing(
+      // final row processing for response
-        rows,
+      rows: await outputProcessing<Row[]>(table, processed, {
-        table!,
+        preserveLinks: true,
-        allTablesMap,
+        squash: true,
-        relationships,
+      }),
-        {
-          sqs: true,
-        }
-      ),
     }
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -112,7 +112,7 @@ describe("external search", () => {
         tableId,
         query: {},
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(10)
       expect(result.rows).toEqual(
@@ -130,7 +130,7 @@ describe("external search", () => {
         query: {},
         fields: ["name", "age"],
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(10)
       expect(result.rows).toEqual(
@@ -157,7 +157,7 @@ describe("external search", () => {
           },
         },
       }
-      const result = await search(searchParams)
+      const result = await search(searchParams, config.table!)
 
       expect(result.rows).toHaveLength(3)
       expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -81,7 +81,7 @@ describe("internal", () => {
           tableId,
           query: {},
         }
-        const result = await search(searchParams)
+        const result = await search(searchParams, config.table!)
 
         expect(result.rows).toHaveLength(10)
         expect(result.rows).toEqual(
@@ -99,7 +99,7 @@ describe("internal", () => {
           query: {},
           fields: ["name", "age"],
         }
-        const result = await search(searchParams)
+        const result = await search(searchParams, config.table!)
 
         expect(result.rows).toHaveLength(10)
         expect(result.rows).toEqual(

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -6,11 +6,12 @@ import {
   Row,
   SearchFilters,
 } from "@budibase/types"
-import { getSQLClient } from "../../../sdk/app/rows/utils"
+import { getSQLClient } from "./utils"
 import { cloneDeep } from "lodash"
-import sdk from "../../../sdk"
+import datasources from "../datasources"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { SqlClient } from "../../../integrations/utils"
+import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
 
 const WRITE_OPERATIONS: Operation[] = [
   Operation.CREATE,
@@ -156,12 +157,19 @@ export default class AliasTables {
   }
 
   async queryWithAliasing(
-    json: QueryJson
+    json: QueryJson,
+    queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
   ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
-    const datasource = await sdk.datasources.get(datasourceId)
+    const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
+    let aliasingEnabled: boolean, datasource: Datasource | undefined
+    if (isSqs) {
+      aliasingEnabled = true
+    } else {
+      datasource = await datasources.get(datasourceId)
+      aliasingEnabled = this.isAliasingEnabled(json, datasource)
+    }
 
-    const aliasingEnabled = this.isAliasingEnabled(json, datasource)
     if (aliasingEnabled) {
       json = cloneDeep(json)
       // run through the query json to update anywhere a table may be used
@@ -207,7 +215,15 @@ export default class AliasTables {
       }
       json.tableAliases = invertedTableAliases
     }
-    const response = await makeExternalQuery(datasource, json)
+
+    let response: DatasourcePlusQueryResponse
+    if (datasource && !isSqs) {
+      response = await makeExternalQuery(datasource, json)
+    } else if (queryFn) {
+      response = await queryFn(json)
+    } else {
+      throw new Error("No supplied method to perform aliased query")
+    }
     if (Array.isArray(response) && aliasingEnabled) {
       return this.reverse(response)
     } else {

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -1,8 +1,19 @@
 import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
-import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
+import {
+  FieldType,
+  RelationshipFieldMetadata,
+  SQLiteDefinition,
+  SQLiteTable,
+  SQLiteTables,
+  SQLiteType,
+  Table,
+} from "@budibase/types"
 import { cloneDeep } from "lodash"
 import tablesSdk from "../"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  generateJunctionTableID,
+} from "../../../../db/utils"
 
 const BASIC_SQLITE_DOC: SQLiteDefinition = {
   _id: SQLITE_DESIGN_DOC_ID,
@@ -36,9 +47,42 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BB_REFERENCE]: SQLiteType.TEXT,
 }
 
-function mapTable(table: Table): { [key: string]: SQLiteType } {
+function buildRelationshipDefinitions(
+  table: Table,
+  relationshipColumn: RelationshipFieldMetadata
+): {
+  tableId: string
+  definition: SQLiteTable
+} {
+  const tableId = table._id!,
+    relatedTableId = relationshipColumn.tableId
+  return {
+    tableId: generateJunctionTableID(tableId, relatedTableId),
+    definition: {
+      ["doc1.rowId"]: SQLiteType.TEXT,
+      ["doc1.tableId"]: SQLiteType.TEXT,
+      ["doc1.fieldName"]: SQLiteType.TEXT,
+      ["doc2.rowId"]: SQLiteType.TEXT,
+      ["doc2.tableId"]: SQLiteType.TEXT,
+      ["doc2.fieldName"]: SQLiteType.TEXT,
+      tableId: SQLiteType.TEXT,
+    },
+  }
+}
+
+// this can generate relationship tables as part of the mapping
+function mapTable(table: Table): SQLiteTables {
+  const tables: SQLiteTables = {}
   const fields: Record<string, SQLiteType> = {}
   for (let [key, column] of Object.entries(table.schema)) {
+    // relationships should be handled differently
+    if (column.type === FieldType.LINK) {
+      const { tableId, definition } = buildRelationshipDefinitions(
+        table,
+        column
+      )
+      tables[tableId] = { fields: definition }
+    }
     if (!FieldTypeMap[column.type]) {
       throw new Error(`Unable to map type "${column.type}" to SQLite type`)
     }
@@ -49,10 +93,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
   CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
     constantMap[col] = SQLiteType.TEXT
   })
-  return {
+  const thisTable: SQLiteTable = {
     ...constantMap,
     ...fields,
   }
+  tables[table._id!] = { fields: thisTable }
+  return tables
 }
 
 // nothing exists, need to iterate though existing tables
@@ -60,8 +106,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
   const definition = cloneDeep(BASIC_SQLITE_DOC)
   for (let table of tables) {
-    definition.sql.tables[table._id!] = {
+    definition.sql.tables = {
-      fields: mapTable(table),
+      ...definition.sql.tables,
+      ...mapTable(table),
     }
   }
   return definition
@@ -75,8 +122,9 @@ export async function addTableToSqlite(table: Table) {
   } catch (err) {
     definition = await buildBaseDefinition()
   }
-  definition.sql.tables[table._id!] = {
+  definition.sql.tables = {
-    fields: mapTable(table),
+    ...definition.sql.tables,
+    ...mapTable(table),
   }
   await db.put(definition)
 }

packages/types/src/documents/app/row.ts

@@ -1,22 +1,111 @@
 import { Document } from "../document"
 
 export enum FieldType {
+  /**
+   * a primitive type, stores a string, called Text within Budibase. This is one of the default
+   * types of Budibase, if an external type is not fully understood, we will treat it as text.
+   */
   STRING = "string",
+  /**
+   * similar to string type, called Long Form Text within Budibase. This is mainly a frontend
+   * orientated type which enables a larger text input area. This can also be used
+   * in conjunction with the 'useRichText' option to support a markdown editor/viewer.
+   */
   LONGFORM = "longform",
+  /**
+   * similar to string type, called Options within Budibase. This works very similarly to
+   * the string type within the backend, but is validated to a list of options. This will
+   * display a <select> input within the builder/client.
+   */
   OPTIONS = "options",
+  /**
+   * a primitive type, stores a number, as a floating point, called Number within Budibase.
+   * this type will always represent numbers as reals/floating point - there is no integer only
+   * type within Budibase.
+   */
   NUMBER = "number",
+  /**
+   * a primitive type, stores a boolean, called Boolean within Budibase. This is often represented
+   * as a toggle or checkbox within forms/grids.
+   */
   BOOLEAN = "boolean",
+  /**
+   * a JSON type, this type is always an array of strings, called Multi-select within Budibase.
+   * This type can be compared to the options type, as it functions similarly, but allows picking
+   * multiple options rather than a single option.
+   */
   ARRAY = "array",
+  /**
+   * a string type, this is always a string when input/returned from the API, called Date/Time within
+   * Budibase. We utilise ISO date strings for representing dates, this type has a range of subtypes
+   * to restrict it to date only, time only and ignore timezone capabilities.
+   */
   DATETIME = "datetime",
+  /**
+   * a JSON type, an array of metadata about files held in object storage, called Attachment List within
+   * Budibase. To utilise this type there is an API for uploading files to Budibase, which returns metadata
+   * that can be stored against columns of this type. Currently this is not supported on external databases.
+   */
   ATTACHMENTS = "attachment",
+  /**
+   * a JSON type, similar to the attachments type, called Attachment within Budibase. This type functions
+   * much the same as the attachment list, but only holds a single attachment metadata as an object.
+   * This simplifies the binding experience of using this column type.
+   */
   ATTACHMENT_SINGLE = "attachment_single",
+  /**
+   * a complex type, called Relationships within Budibase. This is the most complex type of Budibase,
+   * nothing should be stored against rows under link columns; this type simply represents the
+   * relationship between tables as part of the table schema. When rows are input to the Budibase API
+   * relationships to be made are represented as a list of row IDs to link. When rows are returned
+   * from the Budibase API it will contain a list of row IDs and display column values of the related rows.
+   */
   LINK = "link",
+  /**
+   * a complex type, called Formulas within Budibase. This type has two variants, static and dynamic, with
+   * static only being supported against internal tables. Dynamic formulas calculate a provided HBS/JS binding
+   * based on the row context and enrich it when rows are being returned from the API. Static bindings calculate
+   * this when rows are being stored, so that the formula output can be searched upon within the DB.
+   */
   FORMULA = "formula",
+  /**
+   * a complex type, called Auto Column within Budibase. This type has a few variants, with options such as a
+   * date for created at/updated at, an auto ID column with auto-increments as rows are saved and a user
+   * relationship type which stores the created by/updated by user details. These subtypes all depend on the
+   * date, number of link types respectively. There is one case where these will be executed in the browser,
+   * that is part of the initial formula definition, the formula will be live evaluated in the browser.
+   */
   AUTO = "auto",
+  /**
+   * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
+   * type, which will be represented as a JSON object in the row. This type depends on a schema being
+   * provided to make the JSON searchable/bindable, the JSON cannot be fully dynamic.
+   */
   JSON = "json",
+  /**
+   * @deprecated an internal type, this is an old deprecated type which is no longer used - still represented to note it
+   * could appear in very old tables.
+   */
   INTERNAL = "internal",
+  /**
+   * a string type, called Barcode/QR within Budibase. This type is used to denote to forms to that this column
+   * should be filled in using a camera to read a barcode, there is a form component which will be used when this
+   * type is found. The column will contain the contents of any barcode scanned.
+   */
   BARCODEQR = "barcodeqr",
+  /**
+   * a string type, this allows representing very large integers, but they are held/managed within Budibase as
+   * strings. When stored in external databases Budibase will attempt to use a real big integer type and depend
+   * on the database parsing the string to this type as part of saving.
+   */
   BIGINT = "bigint",
+  /**
+   * a JSON type, called User within Budibase. This type is used to represent a link to an internal Budibase
+   * resource, like a user or group, today only users are supported. This type will be represented as an
+   * array of internal resource IDs (e.g. user IDs) within the row - this ID list will be enriched with
+   * the full resources when rows are returned from the API. The full resources can be input to the API, or
+   * an array of resource IDs, the API will squash these down and validate them before saving the row.
+   */
   BB_REFERENCE = "bb_reference",
 }
 

packages/types/src/documents/app/sqlite.ts

@@ -6,17 +6,23 @@ export enum SQLiteType {
   NUMERIC = "NUMERIC",
 }
 
+export type SQLiteTable = Record<
+  string,
+  SQLiteType | { field: string; type: SQLiteType }
+>
+
+export type SQLiteTables = Record<
+  string,
+  {
+    fields: SQLiteTable
+  }
+>
+
 export interface SQLiteDefinition {
   _id: string
   language: string
   sql: {
-    tables: {
+    tables: SQLiteTables
-      [tableName: string]: {
-        fields: {
-          [key: string]: SQLiteType | { field: string; type: SQLiteType }
-        }
-      }
-    }
     options: {
       table_name: string
     }

yarn.lock

@@ -5174,7 +5174,7 @@
   dependencies:
     "@types/node" "*"
 
-"@types/archiver@^6.0.2":
+"@types/archiver@6.0.2":
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/@types/archiver/-/archiver-6.0.2.tgz#0daf8c83359cbde69de1e4b33dcade6a48a929e2"
   integrity sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==