diff --git a/hosting/.env b/hosting/.env index 8a0756c0e3..173d409d04 100644 --- a/hosting/.env +++ b/hosting/.env @@ -17,6 +17,7 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION @@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= -ROLLING_LOG_MAX_SIZE= \ No newline at end of file +ROLLING_LOG_MAX_SIZE= diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9dba5d427c..77f6bd053b 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -42,12 +42,13 @@ services: couchdb-service: container_name: budi-couchdb3-dev restart: on-failure - image: budibase/couchdb + image: budibase/couchdb:v3.2.1-sqs environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" + - "${COUCH_DB_SQS_PORT}:4984" volumes: - couchdb_data:/data diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2da2a77d67..8dbc904643 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,7 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index f8adcbe0ee..d9d299d5fa 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -28,6 +28,7 @@ const DEFAULTS = { PLUGINS_DIR: "/plugins", FORKED_PROCESS_NAME: "main", JS_RUNNER_MEMORY_LIMIT: 64, + COUCH_DB_SQL_URL: "http://localhost:4006", } const QUERY_THREAD_TIMEOUT = @@ -39,6 +40,7 @@ const environment = { // important - prefer app port to generic port PORT: process.env.APP_PORT || process.env.PORT, COUCH_DB_URL: process.env.COUCH_DB_URL, + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL, MINIO_URL: process.env.MINIO_URL, WORKER_URL: process.env.WORKER_URL, AWS_REGION: process.env.AWS_REGION, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 79d9be2348..5dd16f516c 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -1,8 +1,19 @@ import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" -import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types" +import { + FieldType, + RelationshipFieldMetadata, + SQLiteDefinition, + SQLiteTable, + SQLiteTables, + SQLiteType, + Table, +} from "@budibase/types" import { cloneDeep } from "lodash" import tablesSdk from "../" -import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" +import { + CONSTANT_INTERNAL_ROW_COLS, + generateJunctionTableID, +} from "../../../../db/utils" const BASIC_SQLITE_DOC: SQLiteDefinition = { _id: SQLITE_DESIGN_DOC_ID, @@ -36,9 +47,38 @@ const FieldTypeMap: Record = { [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } -function mapTable(table: Table): { [key: string]: SQLiteType } { +function buildRelationshipDefinitions( + table: Table, + relationshipColumn: RelationshipFieldMetadata +): { + tableId: string + definition: SQLiteTable +} { + const tableId = table._id!, + relatedTableId = relationshipColumn.tableId + return { + tableId: generateJunctionTableID(tableId, relatedTableId), + definition: { + doc1: SQLiteType.BLOB, + doc2: SQLiteType.BLOB, + tableId: SQLiteType.TEXT, + }, + } +} + +// this can generate relationship tables as part of the mapping +function mapTable(table: Table): SQLiteTables { + const tables: SQLiteTables = {} const fields: Record = {} for (let [key, column] of Object.entries(table.schema)) { + // relationships should be handled differently + if (column.type === FieldType.LINK) { + const { tableId, definition } = buildRelationshipDefinitions( + table, + column + ) + tables[tableId] = { fields: definition } + } if (!FieldTypeMap[column.type]) { throw new Error(`Unable to map type "${column.type}" to SQLite type`) } @@ -49,10 +89,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } { CONSTANT_INTERNAL_ROW_COLS.forEach(col => { constantMap[col] = SQLiteType.TEXT }) - return { + const thisTable: SQLiteTable = { ...constantMap, ...fields, } + tables[table._id!] = { fields: thisTable } + return tables } // nothing exists, need to iterate though existing tables @@ -60,8 +102,9 @@ async function buildBaseDefinition(): Promise { const tables = await tablesSdk.getAllInternalTables() const definition = cloneDeep(BASIC_SQLITE_DOC) for (let table of tables) { - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } } return definition @@ -75,8 +118,9 @@ export async function addTableToSqlite(table: Table) { } catch (err) { definition = await buildBaseDefinition() } - definition.sql.tables[table._id!] = { - fields: mapTable(table), + definition.sql.tables = { + ...definition.sql.tables, + ...mapTable(table), } await db.put(definition) } diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts index 76c47bbd74..e23a68b336 100644 --- a/packages/types/src/documents/app/sqlite.ts +++ b/packages/types/src/documents/app/sqlite.ts @@ -6,17 +6,23 @@ export enum SQLiteType { NUMERIC = "NUMERIC", } +export type SQLiteTable = Record< + string, + SQLiteType | { field: string; type: SQLiteType } +> + +export type SQLiteTables = Record< + string, + { + fields: SQLiteTable + } +> + export interface SQLiteDefinition { _id: string language: string sql: { - tables: { - [tableName: string]: { - fields: { - [key: string]: SQLiteType | { field: string; type: SQLiteType } - } - } - } + tables: SQLiteTables options: { table_name: string }