Initial attempt to get SQS up and running within BB.

hosting/couchdb/.gitignore

@@ -0,0 +1 @@
+sqlite

hosting/couchdb/Dockerfile

@@ -3,6 +3,7 @@ FROM couchdb:3.2.1
 ENV COUCHDB_USER admin
 ENV COUCHDB_PASSWORD admin
 EXPOSE 5984
+EXPOSE 4984
 
 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
     wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \
@@ -28,8 +29,11 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./
 WORKDIR /opt/couchdb
 ADD couch/vm.args couch/local.ini ./etc/
 
+WORKDIR /opt/sqs
+ADD sqlite/sqs sqlite/better_sqlite3.node ./
+
 WORKDIR /
 ADD build-target-paths.sh .
 ADD runner.sh ./bbcouch-runner.sh
-RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh /opt/sqs/sqs
 CMD ["./bbcouch-runner.sh"]

hosting/couchdb/runner.sh

@@ -8,6 +8,7 @@ chown -R couchdb:couchdb ${DATA_DIR}/couch
 /build-target-paths.sh
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
 sleep 10
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
 curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator

hosting/docker-compose.dev.yaml

@@ -40,15 +40,15 @@ services:
       - PROXY_ADDRESS=host.docker.internal
 
   couchdb-service:
-    # platform: linux/amd64
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb
+    image: couch-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
+      - "4984:4984"
     volumes:
       - couchdb_data:/data
 

packages/backend-core/src/constants/db.ts

@@ -99,3 +99,4 @@ export const APP_PREFIX = DocumentType.APP + SEPARATOR
 export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
+export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -16,6 +16,7 @@ import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
+import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 
 function buildNano(couchInfo: { url: string; cookie: string }) {
   return Nano({
@@ -180,6 +181,21 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.list(params))
   }
 
+  async sql<T>(sql: string): Promise<T> {
+    const dbName = this.name
+    const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
+    const response = await directCouchUrlCall({
+      url: `${this.couchInfo.sqlUrl}/${url}`,
+      method: "POST",
+      cookie: this.couchInfo.cookie,
+      body: sql,
+    })
+    if (response.status > 300) {
+      throw new Error(await response.text())
+    }
+    return (await response.json()) as T
+  }
+
   async query<T>(
     viewName: string,
     params: DatabaseQueryOpts

packages/backend-core/src/db/couch/connections.ts

@@ -25,6 +25,7 @@ export const getCouchInfo = (connection?: string) => {
   const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
   return {
     url: urlInfo.url!,
+    sqlUrl: env.COUCH_DB_SQL_URL,
     auth: {
       username: username,
       password: password,

packages/backend-core/src/db/couch/utils.ts

@@ -30,8 +30,13 @@ export async function directCouchUrlCall({
     },
   }
   if (body && method !== "GET") {
-    params.body = JSON.stringify(body)
+    if (typeof body === "string") {
-    params.headers["Content-Type"] = "application/json"
+      params.body = body
+      params.headers["Content-Type"] = "text/plain"
+    } else {
+      params.body = JSON.stringify(body)
+      params.headers["Content-Type"] = "application/json"
+    }
   }
   return await fetch(checkSlashesInUrl(encodeURI(url)), params)
 }

packages/backend-core/src/environment.ts

@@ -89,6 +89,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/builder/src/helpers/searchFields.js

@@ -4,11 +4,11 @@ import { get } from "svelte/store"
 
 export function getTableFields(linkField) {
   const table = get(tables).list.find(table => table._id === linkField.tableId)
-  if (!table || !table.sql) {
+  if (!table) {
     return []
   }
   const linkFields = getFields(Object.values(table.schema), {
-    allowLinks: false,
+    tableFields: true,
   })
   return linkFields.map(field => ({
     ...field,
@@ -16,11 +16,11 @@ export function getTableFields(linkField) {
   }))
 }
 
-export function getFields(fields, { allowLinks } = { allowLinks: true }) {
+export function getFields(fields, { tableFields }) {
   let filteredFields = fields.filter(
     field => !BannedSearchTypes.includes(field.type)
   )
-  if (allowLinks) {
+  if (!tableFields) {
     const linkFields = fields.filter(field => field.type === "link")
     for (let linkField of linkFields) {
       // only allow one depth of SQL relationship filtering

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -6,7 +6,6 @@ import {
   IncludeRelationship,
   Operation,
   PaginationJson,
-  RelationshipsJson,
   RelationshipTypes,
   Row,
   SearchFilters,
@@ -18,19 +17,21 @@ import {
   breakExternalTableId,
   breakRowIdField,
   convertRowId,
-  generateRowIdField,
-  getPrimaryDisplay,
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
-import { getDatasourceAndQuery } from "./utils"
+import {
+  getDatasourceAndQuery,
+  generateIdForRow,
+  buildExternalRelationships,
+  buildSqlFieldList,
+  sqlOutputProcessing,
+} from "./utils"
 import { FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { isEditableColumn } from "../../../sdk/app/tables/validation"
 
 export interface ManyRelationship {
   tableId?: string
@@ -146,34 +147,6 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   return config
 }
 
-function generateIdForRow(
-  row: Row | undefined,
-  table: Table,
-  isLinked: boolean = false
-): string {
-  const primary = table.primary
-  if (!row || !primary) {
-    return ""
-  }
-  // build id array
-  let idParts = []
-  for (let field of primary) {
-    let fieldValue = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName: field,
-      isLinked,
-    })
-    if (fieldValue) {
-      idParts.push(fieldValue)
-    }
-  }
-  if (idParts.length === 0) {
-    return ""
-  }
-  return generateRowIdField(idParts)
-}
-
 function getEndpoint(tableId: string | undefined, operation: string) {
   if (!tableId) {
     return {}
@@ -186,74 +159,6 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
 }
 
-// need to handle table name + field or just field, depending on if relationships used
-function extractFieldValue({
-  row,
-  tableName,
-  fieldName,
-  isLinked,
-}: {
-  row: Row
-  tableName: string
-  fieldName: string
-  isLinked: boolean
-}) {
-  let value = row[`${tableName}.${fieldName}`]
-  if (value == null && !isLinked) {
-    value = row[fieldName]
-  }
-  return value
-}
-
-function basicProcessing({
-  row,
-  table,
-  isLinked,
-}: {
-  row: Row
-  table: Table
-  isLinked: boolean
-}): Row {
-  const thisRow: Row = {}
-  // filter the row down to what is actually the row (not joined)
-  for (let field of Object.values(table.schema)) {
-    const fieldName = field.name
-
-    const value = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName,
-      isLinked,
-    })
-
-    // all responses include "select col as table.col" so that overlaps are handled
-    if (value != null) {
-      thisRow[fieldName] = value
-    }
-  }
-  thisRow._id = generateIdForRow(row, table, isLinked)
-  thisRow.tableId = table._id
-  thisRow._rev = "rev"
-  return processFormulas(table, thisRow)
-}
-
-function fixArrayTypes(row: Row, table: Table) {
-  for (let [fieldName, schema] of Object.entries(table.schema)) {
-    if (
-      schema.type === FieldTypes.ARRAY &&
-      typeof row[fieldName] === "string"
-    ) {
-      try {
-        row[fieldName] = JSON.parse(row[fieldName])
-      } catch (err) {
-        // couldn't convert back to array, ignore
-        delete row[fieldName]
-      }
-    }
-  }
-  return row
-}
-
 function isOneSide(field: FieldSchema) {
   return (
     field.relationshipType && field.relationshipType.split("-")[0] === "one"
@@ -372,189 +277,6 @@ export class ExternalRequest {
     return { row: newRow, manyRelationships }
   }
 
-  squashRelationshipColumns(
-    table: Table,
-    row: Row,
-    relationships: RelationshipsJson[]
-  ): Row {
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable || !row[relationship.column]) {
-        continue
-      }
-      const display = linkedTable.primaryDisplay
-      for (let key of Object.keys(row[relationship.column])) {
-        let relatedRow: Row = row[relationship.column][key]
-        // add this row as context for the relationship
-        for (let col of Object.values(linkedTable.schema)) {
-          if (col.type === FieldType.LINK && col.tableId === table._id) {
-            relatedRow[col.name] = [row]
-          }
-        }
-        relatedRow = processFormulas(linkedTable, relatedRow)
-        let relatedDisplay
-        if (display) {
-          relatedDisplay = getPrimaryDisplay(relatedRow[display])
-        }
-        row[relationship.column][key] = {
-          primaryDisplay: relatedDisplay || "Invalid display column",
-          _id: relatedRow._id,
-        }
-      }
-    }
-    return row
-  }
-
-  /**
-   * This iterates through the returned rows and works out what elements of the rows
-   * actually match up to another row (based on primary keys) - this is pretty specific
-   * to SQL and the way that SQL relationships are returned based on joins.
-   * This is complicated, but the idea is that when a SQL query returns all the relations
-   * will be separate rows, with all of the data in each row. We have to decipher what comes
-   * from where (which tables) and how to convert that into budibase columns.
-   */
-  updateRelationshipColumns(
-    table: Table,
-    row: Row,
-    rows: { [key: string]: Row },
-    relationships: RelationshipsJson[]
-  ) {
-    const columns: { [key: string]: any } = {}
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable) {
-        continue
-      }
-      const fromColumn = `${table.name}.${relationship.from}`
-      const toColumn = `${linkedTable.name}.${relationship.to}`
-      // this is important when working with multiple relationships
-      // between the same tables, don't want to overlap/multiply the relations
-      if (
-        !relationship.through &&
-        row[fromColumn]?.toString() !== row[toColumn]?.toString()
-      ) {
-        continue
-      }
-
-      let linked = basicProcessing({ row, table: linkedTable, isLinked: true })
-      if (!linked._id) {
-        continue
-      }
-      columns[relationship.column] = linked
-    }
-    for (let [column, related] of Object.entries(columns)) {
-      if (!row._id) {
-        continue
-      }
-      const rowId: string = row._id
-      if (!Array.isArray(rows[rowId][column])) {
-        rows[rowId][column] = []
-      }
-      // make sure relationship hasn't been found already
-      if (
-        !rows[rowId][column].find(
-          (relation: Row) => relation._id === related._id
-        )
-      ) {
-        rows[rowId][column].push(related)
-      }
-    }
-    return rows
-  }
-
-  outputProcessing(
-    rows: Row[] = [],
-    table: Table,
-    relationships: RelationshipsJson[]
-  ) {
-    if (!rows || rows.length === 0 || rows[0].read === true) {
-      return []
-    }
-    let finalRows: { [key: string]: Row } = {}
-    for (let row of rows) {
-      const rowId = generateIdForRow(row, table)
-      row._id = rowId
-      // this is a relationship of some sort
-      if (finalRows[rowId]) {
-        finalRows = this.updateRelationshipColumns(
-          table,
-          row,
-          finalRows,
-          relationships
-        )
-        continue
-      }
-      const thisRow = fixArrayTypes(
-        basicProcessing({ row, table, isLinked: false }),
-        table
-      )
-      if (thisRow._id == null) {
-        throw "Unable to generate row ID for SQL rows"
-      }
-      finalRows[thisRow._id] = thisRow
-      // do this at end once its been added to the final rows
-      finalRows = this.updateRelationshipColumns(
-        table,
-        row,
-        finalRows,
-        relationships
-      )
-    }
-
-    // Process some additional data types
-    let finalRowArray = Object.values(finalRows)
-    finalRowArray = processDates(table, finalRowArray)
-    finalRowArray = processFormulas(table, finalRowArray) as Row[]
-
-    return finalRowArray.map((row: Row) =>
-      this.squashRelationshipColumns(table, row, relationships)
-    )
-  }
-
-  /**
-   * Gets the list of relationship JSON structures based on the columns in the table,
-   * this will be used by the underlying library to build whatever relationship mechanism
-   * it has (e.g. SQL joins).
-   */
-  buildRelationships(table: Table): RelationshipsJson[] {
-    const relationships = []
-    for (let [fieldName, field] of Object.entries(table.schema)) {
-      if (field.type !== FieldTypes.LINK) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      // no table to link to, this is not a valid relationships
-      if (!linkTableName || !this.tables[linkTableName]) {
-        continue
-      }
-      const linkTable = this.tables[linkTableName]
-      if (!table.primary || !linkTable.primary) {
-        continue
-      }
-      const definition: any = {
-        // if no foreign key specified then use the name of the field in other table
-        from: field.foreignKey || table.primary[0],
-        to: field.fieldName,
-        tableName: linkTableName,
-        // need to specify where to put this back into
-        column: fieldName,
-      }
-      if (field.through) {
-        const { tableName: throughTableName } = breakExternalTableId(
-          field.through
-        )
-        definition.through = throughTableName
-        // don't support composite keys for relationships
-        definition.from = field.throughTo || table.primary[0]
-        definition.to = field.throughFrom || linkTable.primary[0]
-        definition.fromPrimary = table.primary[0]
-        definition.toPrimary = linkTable.primary[0]
-      }
-      relationships.push(definition)
-    }
-    return relationships
-  }
-
   /**
    * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
    * information.
@@ -704,41 +426,6 @@ export class ExternalRequest {
     await Promise.all(promises)
   }
 
-  /**
-   * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
-   * you have column overlap in relationships, e.g. we join a few different tables and they all have the
-   * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
-   * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
-   * is more performant and has the added benefit of protecting against this scenario.
-   */
-  buildFields(table: Table, includeRelations: boolean) {
-    function extractRealFields(table: Table, existing: string[] = []) {
-      return Object.entries(table.schema)
-        .filter(
-          column =>
-            column[1].type !== FieldTypes.LINK &&
-            column[1].type !== FieldTypes.FORMULA &&
-            !existing.find((field: string) => field === column[0])
-        )
-        .map(column => `${table.name}.${column[0]}`)
-    }
-    let fields = extractRealFields(table)
-    for (let field of Object.values(table.schema)) {
-      if (field.type !== FieldTypes.LINK || !includeRelations) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      if (linkTableName) {
-        const linkTable = this.tables[linkTableName]
-        if (linkTable) {
-          const linkedFields = extractRealFields(linkTable, fields)
-          fields = fields.concat(linkedFields)
-        }
-      }
-    }
-    return fields
-  }
-
   async run(config: RunConfig) {
     const { operation, tableId } = this
     let { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -777,9 +464,9 @@ export class ExternalRequest {
       }
     }
     filters = buildFilters(id, filters || {}, table)
-    const relationships = this.buildRelationships(table)
+    const relationships = buildExternalRelationships(table, this.tables)
 
-    const includeSqlRelationships =
+    const incRelationships =
       config.includeSqlRelationships === IncludeRelationship.INCLUDE
 
     // clean up row on ingress using schema
@@ -799,7 +486,11 @@ export class ExternalRequest {
       },
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
-        fields: isSql ? this.buildFields(table, includeSqlRelationships) : [],
+        fields: isSql
+          ? buildSqlFieldList(table, this.tables, {
+              relationships: incRelationships,
+            })
+          : [],
       },
       filters,
       sort,
@@ -825,7 +516,12 @@ export class ExternalRequest {
         processed.manyRelationships
       )
     }
-    const output = this.outputProcessing(response, table, relationships)
+    const output = sqlOutputProcessing(
+      response,
+      table,
+      this.tables,
+      relationships
+    )
     // if reading it'll just be an array of rows, return whole thing
     return operation === Operation.READ && Array.isArray(response)
       ? output

packages/server/src/api/controllers/row/internal.ts

@@ -14,8 +14,8 @@ import {
 } from "../../../utilities/rowProcessor"
 import { FieldTypes } from "../../../constants"
 import * as utils from "./utils"
-import { fullSearch, paginatedSearch } from "./internalSearch"
+// import { fullSearch, paginatedSearch } from "./internalSearch"
-import { getGlobalUsersFromMetadata } from "../../../utilities/global"
+// import { getGlobalUsersFromMetadata } from "../../../utilities/global"
 import * as inMemoryViews from "../../../db/inMemoryView"
 import env from "../../../environment"
 import {
@@ -36,6 +36,7 @@ import {
   Row,
   Table,
 } from "@budibase/types"
+import { sqlSearch } from "./internalSql"
 
 import { cleanExportRows } from "./utils"
 
@@ -355,43 +356,44 @@ export async function bulkDestroy(ctx: UserCtx) {
 }
 
 export async function search(ctx: UserCtx) {
-  // Fetch the whole table when running in cypress, as search doesn't work
+  return sqlSearch(ctx)
-  if (!env.COUCH_DB_URL && env.isCypress()) {
+  // // Fetch the whole table when running in cypress, as search doesn't work
-    return { rows: await fetch(ctx) }
+  // if (!env.COUCH_DB_URL && env.isCypress()) {
-  }
+  //   return { rows: await fetch(ctx) }
-
+  // }
-  const { tableId } = ctx.params
+  //
-  const db = context.getAppDB()
+  // const { tableId } = ctx.params
-  const { paginate, query, ...params } = ctx.request.body
+  // const db = context.getAppDB()
-  params.version = ctx.version
+  // const { paginate, query, ...params } = ctx.request.body
-  params.tableId = tableId
+  // params.version = ctx.version
-
+  // params.tableId = tableId
-  let table
+  //
-  if (params.sort && !params.sortType) {
+  // let table
-    table = await db.get(tableId)
+  // if (params.sort && !params.sortType) {
-    const schema = table.schema
+  //   table = await db.get(tableId)
-    const sortField = schema[params.sort]
+  //   const schema = table.schema
-    params.sortType = sortField.type == "number" ? "number" : "string"
+  //   const sortField = schema[params.sort]
-  }
+  //   params.sortType = sortField.type == "number" ? "number" : "string"
-
+  // }
-  let response
+  //
-  if (paginate) {
+  // let response
-    response = await paginatedSearch(query, params)
+  // if (paginate) {
-  } else {
+  //   response = await paginatedSearch(query, params)
-    response = await fullSearch(query, params)
+  // } else {
-  }
+  //   response = await fullSearch(query, params)
-
+  // }
-  // Enrich search results with relationships
+  //
-  if (response.rows && response.rows.length) {
+  // // Enrich search results with relationships
-    // enrich with global users if from users table
+  // if (response.rows && response.rows.length) {
-    if (tableId === InternalTables.USER_METADATA) {
+  //   // enrich with global users if from users table
-      response.rows = await getGlobalUsersFromMetadata(response.rows)
+  //   if (tableId === InternalTables.USER_METADATA) {
-    }
+  //     response.rows = await getGlobalUsersFromMetadata(response.rows)
-    table = table || (await db.get(tableId))
+  //   }
-    response.rows = await outputProcessing(table, response.rows)
+  //   table = table || (await db.get(tableId))
-  }
+  //   response.rows = await outputProcessing(table, response.rows)
-
+  // }
-  return response
+  //
+  // return response
 }
 
 export async function exportRows(ctx: UserCtx) {
@@ -404,7 +406,7 @@ export async function exportRows(ctx: UserCtx) {
   }
   const { columns, query } = ctx.request.body
 
-  let result
+  let result: Row[] = []
   if (rowIds) {
     let response = (
       await db.allDocs({
@@ -413,7 +415,7 @@ export async function exportRows(ctx: UserCtx) {
       })
     ).rows.map(row => row.doc)
 
-    result = await outputProcessing(table, response)
+    result = (await outputProcessing(table, response)) as Row[]
   } else if (query) {
     let searchResponse = await search(ctx)
     result = searchResponse.rows

packages/server/src/api/controllers/row/internalSql.ts

@@ -0,0 +1,158 @@
+import {
+  FieldType,
+  Operation,
+  QueryJson,
+  Row,
+  SearchFilters,
+  SortType,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import SqlQueryBuilder from "../../../integrations/base/sql"
+import { SqlClient } from "../../../integrations/utils"
+import { buildInternalRelationships, sqlOutputProcessing } from "./utils"
+import sdk from "../../../sdk"
+import { context } from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../db/utils"
+
+function buildInternalFieldList(
+  table: Table,
+  tables: Table[],
+  opts: { relationships: boolean } = { relationships: true }
+) {
+  let fieldList: string[] = []
+  fieldList = fieldList.concat(
+    CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
+  )
+  for (let col of Object.values(table.schema)) {
+    const isLink = col.type === FieldType.LINK
+    if (isLink && !opts.relationships) {
+      continue
+    }
+    if (isLink) {
+      const relatedTable = tables.find(table => table._id === col.tableId)!
+      fieldList = fieldList.concat(
+        buildInternalFieldList(relatedTable, tables, { relationships: false })
+      )
+    } else {
+      fieldList.push(`${table._id}.${col.name}`)
+    }
+  }
+  return fieldList
+}
+
+function tableInFilter(name: string) {
+  return `:${name}.`
+}
+
+function cleanupFilters(filters: SearchFilters, tables: Table[]) {
+  for (let filter of Object.values(filters)) {
+    if (typeof filter !== "object") {
+      continue
+    }
+    for (let [key, keyFilter] of Object.entries(filter)) {
+      if (keyFilter === "") {
+        delete filter[key]
+      }
+
+      // relationship, switch to table ID
+      const tableRelated = tables.find(table =>
+        key.includes(tableInFilter(table.originalName!))
+      )
+      if (tableRelated) {
+        filter[
+          key.replace(
+            tableInFilter(tableRelated.originalName!),
+            tableInFilter(tableRelated._id!)
+          )
+        ] = filter[key]
+        delete filter[key]
+      }
+    }
+  }
+  return filters
+}
+
+function buildTableMap(tables: Table[]) {
+  const tableMap: Record<string, Table> = {}
+  for (let table of tables) {
+    // update the table name, should never query by name for SQLite
+    table.originalName = table.name
+    table.name = table._id!
+    tableMap[table._id!] = table
+  }
+  return tableMap
+}
+
+export async function sqlSearch(ctx: UserCtx) {
+  const { tableId } = ctx.params
+  const { paginate, query, ...params } = ctx.request.body
+
+  const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
+  const allTables = await sdk.tables.getAllInternalTables()
+  const allTablesMap = buildTableMap(allTables)
+  const table = allTables.find(table => table._id === tableId)
+  if (!table) {
+    ctx.throw(400, "Unable to find table")
+  }
+
+  const relationships = buildInternalRelationships(table)
+
+  const request: QueryJson = {
+    endpoint: {
+      // not important, we query ourselves
+      datasourceId: "internal",
+      entityId: table._id!,
+      operation: Operation.READ,
+    },
+    filters: cleanupFilters(query, allTables),
+    table,
+    meta: {
+      table,
+      tables: allTablesMap,
+    },
+    resource: {
+      fields: buildInternalFieldList(table, allTables),
+    },
+    relationships,
+  }
+  // make sure only rows returned
+  request.filters!.equal = {
+    ...request.filters?.equal,
+    type: "row",
+  }
+
+  if (params.sort && !params.sortType) {
+    const sortField = table.schema[params.sort]
+    const sortType = sortField.type == "number" ? "number" : "string"
+    request.sort = {
+      [sortField.name]: {
+        direction: params.sortOrder,
+        type: sortType as SortType,
+      },
+    }
+  }
+  if (paginate) {
+    request.paginate = {
+      limit: params.limit,
+      page: params.bookmark,
+    }
+  }
+  let sql = builder._query(request, {
+    disableReturning: true,
+    disablePreparedStatements: true,
+  })
+
+  // quick hack for docIds
+  sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+  sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+  const db = context.getAppDB()
+  const rows = await db.sql<Row[]>(sql)
+
+  return {
+    rows: sqlOutputProcessing(rows, table, allTablesMap, relationships, {
+      internal: true,
+    }),
+  }
+}

packages/server/src/api/controllers/row/utils/basic.ts

@@ -0,0 +1,107 @@
+// need to handle table name + field or just field, depending on if relationships used
+import { Row, Table } from "@budibase/types"
+import { generateRowIdField } from "../../../../integrations/utils"
+import { processFormulas } from "../../../../utilities/rowProcessor"
+import { FieldTypes } from "../../../../constants"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+
+function extractFieldValue({
+  row,
+  tableName,
+  fieldName,
+  isLinked,
+}: {
+  row: Row
+  tableName: string
+  fieldName: string
+  isLinked: boolean
+}) {
+  let value = row[`${tableName}.${fieldName}`]
+  if (value == null && !isLinked) {
+    value = row[fieldName]
+  }
+  return value
+}
+
+export function generateIdForRow(
+  row: Row | undefined,
+  table: Table,
+  isLinked: boolean = false
+): string {
+  const primary = table.primary
+  if (!row || !primary) {
+    return ""
+  }
+  // build id array
+  let idParts = []
+  for (let field of primary) {
+    let fieldValue = extractFieldValue({
+      row,
+      tableName: table.name,
+      fieldName: field,
+      isLinked,
+    })
+    if (fieldValue) {
+      idParts.push(fieldValue)
+    }
+  }
+  if (idParts.length === 0) {
+    return ""
+  }
+  return generateRowIdField(idParts)
+}
+
+export function basicProcessing({
+  row,
+  table,
+  isLinked,
+  internal,
+}: {
+  row: Row
+  table: Table
+  isLinked: boolean
+  internal?: boolean
+}): Row {
+  let thisRow: Row = {}
+  // filter the row down to what is actually the row (not joined)
+  let toIterate = Object.keys(table.schema)
+  if (internal) {
+    toIterate = toIterate.concat(CONSTANT_INTERNAL_ROW_COLS)
+  }
+  for (let fieldName of toIterate) {
+    const value = extractFieldValue({
+      row,
+      tableName: internal ? table._id! : table.name,
+      fieldName,
+      isLinked,
+    })
+
+    // all responses include "select col as table.col" so that overlaps are handled
+    if (value != null) {
+      thisRow[fieldName] = value
+    }
+  }
+  if (!internal) {
+    thisRow._id = generateIdForRow(row, table, isLinked)
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+  }
+  return processFormulas(table, thisRow)
+}
+
+export function fixArrayTypes(row: Row, table: Table) {
+  for (let [fieldName, schema] of Object.entries(table.schema)) {
+    if (
+      schema.type === FieldTypes.ARRAY &&
+      typeof row[fieldName] === "string"
+    ) {
+      try {
+        row[fieldName] = JSON.parse(row[fieldName])
+      } catch (err) {
+        // couldn't convert back to array, ignore
+        delete row[fieldName]
+      }
+    }
+  }
+  return row
+}

packages/server/src/api/controllers/row/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./basic"
+export * from "./sqlUtils"
+export * from "./utils"

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -0,0 +1,224 @@
+import { FieldType, RelationshipsJson, Row, Table } from "@budibase/types"
+import { processFormulas } from "../../../../utilities/rowProcessor"
+import {
+  breakExternalTableId,
+  getPrimaryDisplay,
+} from "../../../../integrations/utils"
+import { basicProcessing } from "./basic"
+import { generateJunctionTableID } from "../../../../db/utils"
+
+type TableMap = Record<string, Table>
+
+export function squashRelationshipColumns(
+  table: Table,
+  tables: TableMap,
+  row: Row,
+  relationships: RelationshipsJson[]
+): Row {
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable || !row[relationship.column]) {
+      continue
+    }
+    const display = linkedTable.primaryDisplay
+    for (let key of Object.keys(row[relationship.column])) {
+      let relatedRow: Row = row[relationship.column][key]
+      // add this row as context for the relationship
+      for (let col of Object.values(linkedTable.schema)) {
+        if (col.type === FieldType.LINK && col.tableId === table._id) {
+          relatedRow[col.name] = [row]
+        }
+      }
+      relatedRow = processFormulas(linkedTable, relatedRow)
+      let relatedDisplay
+      if (display) {
+        relatedDisplay = getPrimaryDisplay(relatedRow[display])
+      }
+      row[relationship.column][key] = {
+        primaryDisplay: relatedDisplay || "Invalid display column",
+        _id: relatedRow._id,
+      }
+    }
+  }
+  return row
+}
+
+/**
+ * This iterates through the returned rows and works out what elements of the rows
+ * actually match up to another row (based on primary keys) - this is pretty specific
+ * to SQL and the way that SQL relationships are returned based on joins.
+ * This is complicated, but the idea is that when a SQL query returns all the relations
+ * will be separate rows, with all of the data in each row. We have to decipher what comes
+ * from where (which tables) and how to convert that into budibase columns.
+ */
+export function updateRelationshipColumns(
+  table: Table,
+  tables: TableMap,
+  row: Row,
+  rows: { [key: string]: Row },
+  relationships: RelationshipsJson[],
+  opts?: { internal?: boolean }
+) {
+  const columns: { [key: string]: any } = {}
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable) {
+      continue
+    }
+    const fromColumn = `${table.name}.${relationship.from}`
+    const toColumn = `${linkedTable.name}.${relationship.to}`
+    // this is important when working with multiple relationships
+    // between the same tables, don't want to overlap/multiply the relations
+    if (
+      !relationship.through &&
+      row[fromColumn]?.toString() !== row[toColumn]?.toString()
+    ) {
+      continue
+    }
+
+    let linked = basicProcessing({
+      row,
+      table: linkedTable,
+      isLinked: true,
+      internal: opts?.internal,
+    })
+    if (!linked._id) {
+      continue
+    }
+    columns[relationship.column] = linked
+  }
+  for (let [column, related] of Object.entries(columns)) {
+    let rowId: string = row._id!
+    if (opts?.internal) {
+      const { _id } = basicProcessing({
+        row,
+        table,
+        isLinked: false,
+        internal: opts?.internal,
+      })
+      rowId = _id!
+    }
+    if (!rowId) {
+      continue
+    }
+    if (!Array.isArray(rows[rowId][column])) {
+      rows[rowId][column] = []
+    }
+    // make sure relationship hasn't been found already
+    if (
+      !rows[rowId][column].find((relation: Row) => relation._id === related._id)
+    ) {
+      rows[rowId][column].push(related)
+    }
+  }
+  return rows
+}
+
+/**
+ * Gets the list of relationship JSON structures based on the columns in the table,
+ * this will be used by the underlying library to build whatever relationship mechanism
+ * it has (e.g. SQL joins).
+ */
+export function buildExternalRelationships(
+  table: Table,
+  tables: TableMap
+): RelationshipsJson[] {
+  const relationships = []
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (field.type !== FieldType.LINK) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    // no table to link to, this is not a valid relationships
+    if (!linkTableName || !tables[linkTableName]) {
+      continue
+    }
+    const linkTable = tables[linkTableName]
+    if (!table.primary || !linkTable.primary) {
+      continue
+    }
+    const definition: any = {
+      // if no foreign key specified then use the name of the field in other table
+      from: field.foreignKey || table.primary[0],
+      to: field.fieldName,
+      tableName: linkTableName,
+      // need to specify where to put this back into
+      column: fieldName,
+    }
+    if (field.through) {
+      const { tableName: throughTableName } = breakExternalTableId(
+        field.through
+      )
+      definition.through = throughTableName
+      // don't support composite keys for relationships
+      definition.from = field.throughTo || table.primary[0]
+      definition.to = field.throughFrom || linkTable.primary[0]
+      definition.fromPrimary = table.primary[0]
+      definition.toPrimary = linkTable.primary[0]
+    }
+    relationships.push(definition)
+  }
+  return relationships
+}
+
+export function buildInternalRelationships(table: Table): RelationshipsJson[] {
+  const relationships: RelationshipsJson[] = []
+  const links = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  const tableId = table._id!
+  for (let link of links) {
+    const linkTableId = link.tableId!
+    const junctionTableId = generateJunctionTableID(tableId, linkTableId)
+    const isFirstTable = tableId > linkTableId
+    relationships.push({
+      through: junctionTableId,
+      column: link.name,
+      tableName: linkTableId,
+      fromPrimary: "_id",
+      to: isFirstTable ? "doc2.rowId" : "doc1.rowId",
+      from: isFirstTable ? "doc1.rowId" : "doc2.rowId",
+      toPrimary: "_id",
+    })
+  }
+  return relationships
+}
+
+/**
+ * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+ * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+ * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+ * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+ * is more performant and has the added benefit of protecting against this scenario.
+ */
+export function buildSqlFieldList(
+  table: Table,
+  tables: TableMap,
+  opts?: { relationships: boolean }
+) {
+  function extractRealFields(table: Table, existing: string[] = []) {
+    return Object.entries(table.schema)
+      .filter(
+        column =>
+          column[1].type !== FieldType.LINK &&
+          column[1].type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === column[0])
+      )
+      .map(column => `${table.name}.${column[0]}`)
+  }
+  let fields = extractRealFields(table)
+  for (let field of Object.values(table.schema)) {
+    if (field.type !== FieldType.LINK || !opts?.relationships) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    if (linkTableName) {
+      const linkTable = tables[linkTableName]
+      if (linkTable) {
+        const linkedFields = extractRealFields(linkTable, fields)
+        fields = fields.concat(linkedFields)
+      }
+    }
+  }
+  return fields
+}

packages/server/src/api/controllers/row/utils/utils.ts

@@ -1,11 +1,26 @@
-import { InternalTables } from "../../../db/utils"
+import { InternalTables } from "../../../../db/utils"
-import * as userController from "../user"
+import * as userController from "../../user"
-import { FieldTypes } from "../../../constants"
+import { FieldTypes } from "../../../../constants"
 import { context } from "@budibase/backend-core"
-import { makeExternalQuery } from "../../../integrations/base/query"
+import { makeExternalQuery } from "../../../../integrations/base/query"
-import { FieldType, Row, Table, UserCtx } from "@budibase/types"
+import {
-import { Format } from "../view/exporters"
+  FieldType,
-import sdk from "../../../sdk"
+  RelationshipsJson,
+  Row,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import { Format } from "../../view/exporters"
+import sdk from "../../../../sdk"
+import {
+  processDates,
+  processFormulas,
+} from "../../../../utilities/rowProcessor"
+import {
+  squashRelationshipColumns,
+  updateRelationshipColumns,
+} from "./sqlUtils"
+import { basicProcessing, generateIdForRow, fixArrayTypes } from "./basic"
 
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
@@ -177,3 +192,65 @@ export function getTableId(ctx: any) {
     return ctx.params.viewName
   }
 }
+
+export function sqlOutputProcessing(
+  rows: Row[] = [],
+  table: Table,
+  tables: Record<string, Table>,
+  relationships: RelationshipsJson[],
+  opts?: { internal?: boolean }
+) {
+  if (!rows || rows.length === 0 || rows[0].read === true) {
+    return []
+  }
+  let finalRows: { [key: string]: Row } = {}
+  for (let row of rows) {
+    let rowId = row._id
+    if (!rowId) {
+      rowId = generateIdForRow(row, table)
+      row._id = rowId
+    }
+    // this is a relationship of some sort
+    if (finalRows[rowId]) {
+      finalRows = updateRelationshipColumns(
+        table,
+        tables,
+        row,
+        finalRows,
+        relationships
+      )
+      continue
+    }
+    const thisRow = fixArrayTypes(
+      basicProcessing({
+        row,
+        table,
+        isLinked: false,
+        internal: opts?.internal,
+      }),
+      table
+    )
+    if (thisRow._id == null) {
+      throw "Unable to generate row ID for SQL rows"
+    }
+    finalRows[thisRow._id] = thisRow
+    // do this at end once its been added to the final rows
+    finalRows = updateRelationshipColumns(
+      table,
+      tables,
+      row,
+      finalRows,
+      relationships,
+      opts
+    )
+  }
+
+  // Process some additional data types
+  let finalRowArray = Object.values(finalRows)
+  finalRowArray = processDates(table, finalRowArray)
+  finalRowArray = processFormulas(table, finalRowArray) as Row[]
+
+  return finalRowArray.map((row: Row) =>
+    squashRelationshipColumns(table, tables, row, relationships)
+  )
+}

packages/server/src/api/controllers/table/sqlite.ts

@@ -0,0 +1,75 @@
+import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
+import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
+import { cloneDeep } from "lodash"
+import sdk from "../../../sdk"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../db/utils"
+
+const BASIC_SQLITE_DOC: SQLiteDefinition = {
+  _id: SQLITE_DESIGN_DOC_ID,
+  language: "sqlite",
+  sql: {
+    tables: {},
+    options: {
+      table_name: "tableId",
+    },
+  },
+}
+
+const FieldTypeMap: Record<FieldType, SQLiteType> = {
+  [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
+  [FieldType.DATETIME]: SQLiteType.TEXT,
+  [FieldType.FORMULA]: SQLiteType.TEXT,
+  [FieldType.LONGFORM]: SQLiteType.TEXT,
+  [FieldType.NUMBER]: SQLiteType.REAL,
+  [FieldType.STRING]: SQLiteType.TEXT,
+  [FieldType.AUTO]: SQLiteType.TEXT,
+  [FieldType.JSON]: SQLiteType.BLOB,
+  [FieldType.OPTIONS]: SQLiteType.BLOB,
+  [FieldType.INTERNAL]: SQLiteType.BLOB,
+  [FieldType.BARCODEQR]: SQLiteType.BLOB,
+  [FieldType.ATTACHMENT]: SQLiteType.BLOB,
+  [FieldType.ARRAY]: SQLiteType.BLOB,
+  [FieldType.LINK]: SQLiteType.BLOB,
+}
+
+function mapTable(table: Table): { [key: string]: SQLiteType } {
+  const fields: Record<string, SQLiteType> = {}
+  for (let [key, column] of Object.entries(table.schema)) {
+    fields[key] = FieldTypeMap[column.type]
+  }
+  // there are some extra columns to map - add these in
+  const constantMap: Record<string, SQLiteType> = {}
+  CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
+    constantMap[col] = SQLiteType.TEXT
+  })
+  return {
+    ...constantMap,
+    ...fields,
+  }
+}
+
+// nothing exists, need to iterate though existing tables
+async function buildBaseDefinition(): Promise<SQLiteDefinition> {
+  const tables = await sdk.tables.getAllInternalTables()
+  const definition = cloneDeep(BASIC_SQLITE_DOC)
+  for (let table of tables) {
+    definition.sql.tables[table._id!] = {
+      fields: mapTable(table),
+    }
+  }
+  return definition
+}
+
+export async function addTableToSqlite(table: Table) {
+  const db = context.getAppDB()
+  let definition: SQLiteDefinition
+  try {
+    definition = await db.get(SQLITE_DESIGN_DOC_ID)
+  } catch (err) {
+    definition = await buildBaseDefinition()
+  }
+  definition.sql.tables[table._id!] = {
+    fields: mapTable(table),
+  }
+  await db.put(definition)
+}

packages/server/src/api/controllers/table/utils.ts

@@ -27,6 +27,7 @@ import {
   SourceName,
   Table,
 } from "@budibase/types"
+import { addTableToSqlite } from "./sqlite"
 
 export async function clearColumns(table: any, columnNames: any) {
   const db: Database = context.getAppDB()
@@ -293,6 +294,7 @@ class TableSaveFunctions {
   async after(table: any) {
     table = await handleSearchIndexes(table)
     table = await handleDataImport(this.user, table, this.importRows)
+    await addTableToSqlite(table)
     return table
   }
 

packages/server/src/db/linkedRows/LinkDocument.ts

@@ -1,4 +1,4 @@
-import { generateLinkID } from "../utils"
+import { generateLinkID, generateJunctionTableID } from "../utils"
 import { FieldTypes } from "../../constants"
 import { LinkDocument } from "@budibase/types"
 
@@ -17,6 +17,7 @@ import { LinkDocument } from "@budibase/types"
 class LinkDocumentImpl implements LinkDocument {
   _id: string
   type: string
+  tableId: string
   doc1: {
     rowId: string
     fieldName: string
@@ -44,16 +45,20 @@ class LinkDocumentImpl implements LinkDocument {
       fieldName2
     )
     this.type = FieldTypes.LINK
-    this.doc1 = {
+    this.tableId = generateJunctionTableID(tableId1, tableId2)
+    const docA = {
       tableId: tableId1,
       fieldName: fieldName1,
       rowId: rowId1,
     }
-    this.doc2 = {
+    const docB = {
       tableId: tableId2,
       fieldName: fieldName2,
       rowId: rowId2,
     }
+    // have to determine which one will be doc1 - very important for SQL linking
+    this.doc1 = docA.tableId > docB.tableId ? docA : docB
+    this.doc2 = docA.tableId > docB.tableId ? docB : docA
   }
 }
 

packages/server/src/db/utils.ts

@@ -1,5 +1,6 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
+import { SQLiteType } from "@budibase/types"
 
 type Optional = string | null
 
@@ -43,6 +44,14 @@ export const getUserMetadataParams = dbCore.getUserMetadataParams
 export const generateUserMetadataID = dbCore.generateUserMetadataID
 export const getGlobalIDFromUserMetadataID =
   dbCore.getGlobalIDFromUserMetadataID
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+]
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@@ -262,6 +271,12 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
+export function generateJunctionTableID(tableId1: string, tableId2: string) {
+  const first = tableId1 > tableId2 ? tableId1 : tableId2
+  const second = tableId1 > tableId2 ? tableId2 : tableId1
+  return `${first}${SEPARATOR}${second}`
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */

packages/server/src/definitions/datasource.ts

@@ -6,4 +6,5 @@
 
 export interface QueryOptions {
   disableReturning?: boolean
+  disablePreparedStatements?: boolean
 }

packages/server/src/integrations/base/sql.ts

@@ -564,8 +564,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
         throw `Operation type is not supported by SQL query builder`
     }
 
-    // @ts-ignore
+    if (opts?.disablePreparedStatements) {
-    return query.toSQL().toNative()
+      return query.toString()
+    } else {
+      // @ts-ignore
+      return query.toSQL().toNative()
+    }
   }
 
   async getReturningRow(queryFn: Function, json: QueryJson) {

packages/server/src/integrations/utils.ts

@@ -74,6 +74,7 @@ export enum SqlClient {
   POSTGRES = "pg",
   MY_SQL = "mysql2",
   ORACLE = "oracledb",
+  SQL_LITE = "better-sqlite3",
 }
 
 export function isExternalTable(tableId: string) {

packages/types/src/documents/app/index.ts

@@ -14,3 +14,4 @@ export * from "./backup"
 export * from "./webhook"
 export * from "./links"
 export * from "./component"
+export * from "./sqlite"

packages/types/src/documents/app/links.ts

@@ -2,6 +2,7 @@ import { Document } from "../document"
 
 export interface LinkDocument extends Document {
   type: string
+  tableId: string
   doc1: {
     rowId: string
     fieldName: string

packages/types/src/documents/app/sqlite.ts

@@ -0,0 +1,24 @@
+export enum SQLiteType {
+  REAL = "REAL",
+  TEXT = "VARCHAR",
+  INT = "INTEGER",
+  BLOB = "BLOB",
+  NUMERIC = "NUMERIC",
+}
+
+export interface SQLiteDefinition {
+  _id: string
+  language: string
+  sql: {
+    tables: {
+      [tableName: string]: {
+        fields: {
+          [key: string]: SQLiteType | { field: string; type: SQLiteType }
+        }
+      }
+    }
+    options: {
+      table_name: string
+    }
+  }
+}

packages/types/src/documents/app/table.ts

@@ -74,6 +74,7 @@ export interface Table extends Document {
   type?: string
   views?: { [key: string]: View }
   name: string
+  originalName?: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string

packages/types/src/sdk/db.ts

@@ -100,6 +100,7 @@ export interface Database {
   ): Promise<Nano.DocumentInsertResponse>
   bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
   allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>>
+  sql<T>(sql: string): Promise<T>
   query<T>(
     viewName: string,
     params: DatabaseQueryOpts