Initial implementation of generating SQS junction table definitions.

hosting/.env

@@ -17,6 +17,7 @@ APP_PORT=4002
 WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
+COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
 WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
@@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
 
 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
 PLUGINS_DIR=
-ROLLING_LOG_MAX_SIZE=
+ROLLING_LOG_MAX_SIZE=

hosting/docker-compose.dev.yaml

@@ -42,12 +42,13 @@ services:
   couchdb-service:
     container_name: budi-couchdb3-dev
     restart: on-failure
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.2.1-sqs
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
       - "${COUCH_DB_PORT}:5984"
+      - "${COUCH_DB_SQS_PORT}:4984"
     volumes:
       - couchdb_data:/data
 

packages/backend-core/src/environment.ts

@@ -107,7 +107,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/server/src/environment.ts

@@ -28,6 +28,7 @@ const DEFAULTS = {
   PLUGINS_DIR: "/plugins",
   FORKED_PROCESS_NAME: "main",
   JS_RUNNER_MEMORY_LIMIT: 64,
+  COUCH_DB_SQL_URL: "http://localhost:4006",
 }
 
 const QUERY_THREAD_TIMEOUT =
@@ -39,6 +40,7 @@ const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
   AWS_REGION: process.env.AWS_REGION,

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -1,8 +1,19 @@
 import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
-import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
+import {
+  FieldType,
+  RelationshipFieldMetadata,
+  SQLiteDefinition,
+  SQLiteTable,
+  SQLiteTables,
+  SQLiteType,
+  Table,
+} from "@budibase/types"
 import { cloneDeep } from "lodash"
 import tablesSdk from "../"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  generateJunctionTableID,
+} from "../../../../db/utils"
 
 const BASIC_SQLITE_DOC: SQLiteDefinition = {
   _id: SQLITE_DESIGN_DOC_ID,
@@ -36,9 +47,38 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BB_REFERENCE]: SQLiteType.TEXT,
 }
 
-function mapTable(table: Table): { [key: string]: SQLiteType } {
+function buildRelationshipDefinitions(
+  table: Table,
+  relationshipColumn: RelationshipFieldMetadata
+): {
+  tableId: string
+  definition: SQLiteTable
+} {
+  const tableId = table._id!,
+    relatedTableId = relationshipColumn.tableId
+  return {
+    tableId: generateJunctionTableID(tableId, relatedTableId),
+    definition: {
+      doc1: SQLiteType.BLOB,
+      doc2: SQLiteType.BLOB,
+      tableId: SQLiteType.TEXT,
+    },
+  }
+}
+
+// this can generate relationship tables as part of the mapping
+function mapTable(table: Table): SQLiteTables {
+  const tables: SQLiteTables = {}
   const fields: Record<string, SQLiteType> = {}
   for (let [key, column] of Object.entries(table.schema)) {
+    // relationships should be handled differently
+    if (column.type === FieldType.LINK) {
+      const { tableId, definition } = buildRelationshipDefinitions(
+        table,
+        column
+      )
+      tables[tableId] = { fields: definition }
+    }
     if (!FieldTypeMap[column.type]) {
       throw new Error(`Unable to map type "${column.type}" to SQLite type`)
     }
@@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
   CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
     constantMap[col] = SQLiteType.TEXT
   })
-  return {
+  const thisTable: SQLiteTable = {
     ...constantMap,
     ...fields,
   }
+  tables[table._id!] = { fields: thisTable }
+  return tables
 }
 
 // nothing exists, need to iterate though existing tables
@@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
   const definition = cloneDeep(BASIC_SQLITE_DOC)
   for (let table of tables) {
-    definition.sql.tables[table._id!] = {
+    definition.sql.tables = {
-      fields: mapTable(table),
+      ...definition.sql.tables,
+      ...mapTable(table),
     }
   }
   return definition
@@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) {
   } catch (err) {
     definition = await buildBaseDefinition()
   }
-  definition.sql.tables[table._id!] = {
+  definition.sql.tables = {
-    fields: mapTable(table),
+    ...definition.sql.tables,
+    ...mapTable(table),
   }
   await db.put(definition)
 }

packages/types/src/documents/app/sqlite.ts

@@ -6,17 +6,23 @@ export enum SQLiteType {
   NUMERIC = "NUMERIC",
 }
 
+export type SQLiteTable = Record<
+  string,
+  SQLiteType | { field: string; type: SQLiteType }
+>
+
+export type SQLiteTables = Record<
+  string,
+  {
+    fields: SQLiteTable
+  }
+>
+
 export interface SQLiteDefinition {
   _id: string
   language: string
   sql: {
-    tables: {
+    tables: SQLiteTables
-      [tableName: string]: {
-        fields: {
-          [key: string]: SQLiteType | { field: string; type: SQLiteType }
-        }
-      }
-    }
     options: {
       table_name: string
     }