From 164c5594e343324e37a079ff97a2e68fbea7cb8c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 1 Feb 2021 18:08:06 +0000 Subject: [PATCH 1/5] Fixing attachment issue in self hosting, urls are enriched on way out to point directly to MINIO. --- packages/server/src/api/controllers/row.js | 46 ++++++++++++++++--- .../src/api/controllers/static/index.js | 3 +- packages/server/src/constants/index.js | 1 + 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 6cf094cd1f..8e33be1f22 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -10,6 +10,8 @@ const { } = require("../../db/utils") const usersController = require("./user") const { coerceRowValues } = require("../../utilities") +const env = require("../../environment") +const { OBJ_STORE_DIRECTORY } = require("../../constants") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -51,6 +53,32 @@ async function findRow(db, appId, tableId, rowId) { return row } +/** + * This function "enriches" the input rows with anything they are supposed to contain, for example + * link records or attachment links. + */ +async function enrichRows(appId, table, rows) { + // attach any linked row information + const enriched = await linkRows.attachLinkInfo(appId, rows) + // update the attachments URL depending on hosting + if (env.CLOUD && env.SELF_HOSTED) { + for (let [property, column] of Object.entries(table.schema)) { + if (column.type === "attachment") { + for (let row of enriched) { + if (row[property] == null || row[property].length === 0) { + continue + } + row[property].forEach(attachment => { + attachment.url = `${OBJ_STORE_DIRECTORY}/${appId}/${attachment.url}` + attachment.url = attachment.url.replace("//", "/") + }) + } + } + } + } + return enriched +} + exports.patch = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) @@ -190,7 +218,8 @@ exports.fetchView = async function(ctx) { if (!calculation) { response.rows = response.rows.map(row => row.doc) - ctx.body = await linkRows.attachLinkInfo(appId, response.rows) + const table = await db.get(ctx.params.tableId) + ctx.body = await enrichRows(appId, table, response.rows) } if (calculation === CALCULATION_TYPES.STATS) { @@ -217,14 +246,15 @@ exports.fetchView = async function(ctx) { exports.fetchTableRows = async function(ctx) { const appId = ctx.user.appId + const db = new CouchDB(appId) // special case for users, fetch through the user controller - let rows + let rows, table = await db.get(ctx.params.tableId) if (ctx.params.tableId === ViewNames.USERS) { await usersController.fetch(ctx) rows = ctx.body } else { - const db = new CouchDB(appId) + const response = await db.allDocs( getRowParams(ctx.params.tableId, null, { include_docs: true, @@ -232,15 +262,16 @@ exports.fetchTableRows = async function(ctx) { ) rows = response.rows.map(row => row.doc) } - ctx.body = await linkRows.attachLinkInfo(appId, rows) + ctx.body = await enrichRows(appId, table, rows) } exports.find = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) try { + const table = await db.get(ctx.params.tableId) const row = await findRow(db, appId, ctx.params.tableId, ctx.params.rowId) - ctx.body = await linkRows.attachLinkInfo(appId, row) + ctx.body = await enrichRows(appId, table, row) } catch (err) { ctx.throw(400, err) } @@ -325,9 +356,10 @@ exports.fetchEnrichedRow = async function(ctx) { keys: linkVals.map(linkVal => linkVal.id), }) // need to include the IDs in these rows for any links they may have - let linkedRows = await linkRows.attachLinkInfo( + let linkedRows = await enrichRows( appId, - response.rows.map(row => row.doc) + table, + response.rows.map(row => row.doc), ) // insert the link rows in the correct place throughout the main row for (let fieldName of Object.keys(table.schema)) { diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index 770cbaf088..bf01314795 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -17,11 +17,12 @@ const CouchDB = require("../../../db") const setBuilderToken = require("../../../utilities/builder/setBuilderToken") const fileProcessor = require("../../../utilities/fileProcessor") const env = require("../../../environment") +const { OBJ_STORE_DIRECTORY } = require("../../../constants") function objectStoreUrl() { if (env.SELF_HOSTED) { // can use a relative url for this as all goes through the proxy (this is hosted in minio) - return `/app-assets/assets` + return OBJ_STORE_DIRECTORY } else { return "https://cdn.app.budi.live/assets" } diff --git a/packages/server/src/constants/index.js b/packages/server/src/constants/index.js index 5a97a62af6..2e18de98af 100644 --- a/packages/server/src/constants/index.js +++ b/packages/server/src/constants/index.js @@ -43,3 +43,4 @@ exports.AuthTypes = AuthTypes exports.USERS_TABLE_SCHEMA = USERS_TABLE_SCHEMA exports.BUILDER_CONFIG_DB = "builder-config-db" exports.HOSTING_DOC = "hosting-doc" +exports.OBJ_STORE_DIRECTORY = "/app-assets/assets" From c9690f730c5aad23f02f0d489118152e661dc264 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Feb 2021 11:46:10 +0000 Subject: [PATCH 2/5] Added some unit tests for the enrichment process of rows, in the process found some issues with linking a table to itself, so fixed those so that we can do that in the future if desired. --- packages/server/src/api/controllers/row.js | 36 ++--------- packages/server/src/api/controllers/table.js | 5 +- .../src/api/routes/tests/couchTestUtils.js | 59 ++++++++++++------- .../server/src/api/routes/tests/row.spec.js | 44 ++++++++++++++ .../src/db/linkedRows/LinkController.js | 6 +- .../server/src/db/linkedRows/linkUtils.js | 13 ++-- packages/server/src/utilities/index.js | 33 +++++++++++ 7 files changed, 137 insertions(+), 59 deletions(-) diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 8e33be1f22..6a57d54bea 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -9,9 +9,7 @@ const { ViewNames, } = require("../../db/utils") const usersController = require("./user") -const { coerceRowValues } = require("../../utilities") -const env = require("../../environment") -const { OBJ_STORE_DIRECTORY } = require("../../constants") +const { coerceRowValues, enrichRows } = require("../../utilities") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -53,32 +51,6 @@ async function findRow(db, appId, tableId, rowId) { return row } -/** - * This function "enriches" the input rows with anything they are supposed to contain, for example - * link records or attachment links. - */ -async function enrichRows(appId, table, rows) { - // attach any linked row information - const enriched = await linkRows.attachLinkInfo(appId, rows) - // update the attachments URL depending on hosting - if (env.CLOUD && env.SELF_HOSTED) { - for (let [property, column] of Object.entries(table.schema)) { - if (column.type === "attachment") { - for (let row of enriched) { - if (row[property] == null || row[property].length === 0) { - continue - } - row[property].forEach(attachment => { - attachment.url = `${OBJ_STORE_DIRECTORY}/${appId}/${attachment.url}` - attachment.url = attachment.url.replace("//", "/") - }) - } - } - } - } - return enriched -} - exports.patch = async function(ctx) { const appId = ctx.user.appId const db = new CouchDB(appId) @@ -249,12 +221,12 @@ exports.fetchTableRows = async function(ctx) { const db = new CouchDB(appId) // special case for users, fetch through the user controller - let rows, table = await db.get(ctx.params.tableId) + let rows, + table = await db.get(ctx.params.tableId) if (ctx.params.tableId === ViewNames.USERS) { await usersController.fetch(ctx) rows = ctx.body } else { - const response = await db.allDocs( getRowParams(ctx.params.tableId, null, { include_docs: true, @@ -359,7 +331,7 @@ exports.fetchEnrichedRow = async function(ctx) { let linkedRows = await enrichRows( appId, table, - response.rows.map(row => row.doc), + response.rows.map(row => row.doc) ) // insert the link rows in the correct place throughout the main row for (let fieldName of Object.keys(table.schema)) { diff --git a/packages/server/src/api/controllers/table.js b/packages/server/src/api/controllers/table.js index 488809d156..1dd53e2277 100644 --- a/packages/server/src/api/controllers/table.js +++ b/packages/server/src/api/controllers/table.js @@ -108,7 +108,7 @@ exports.save = async function(ctx) { } // update linked rows - await linkRows.updateLinks({ + const linkResp = await linkRows.updateLinks({ appId, eventType: oldTable ? linkRows.EventType.TABLE_UPDATED @@ -116,6 +116,9 @@ exports.save = async function(ctx) { table: tableToSave, oldTable: oldTable, }) + if (linkResp != null && linkResp._rev) { + tableToSave._rev = linkResp._rev + } // don't perform any updates until relationships have been // checked by the updateLinks function diff --git a/packages/server/src/api/routes/tests/couchTestUtils.js b/packages/server/src/api/routes/tests/couchTestUtils.js index 463d88e637..adf75555ff 100644 --- a/packages/server/src/api/routes/tests/couchTestUtils.js +++ b/packages/server/src/api/routes/tests/couchTestUtils.js @@ -4,6 +4,7 @@ const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles") const packageJson = require("../../../../package") const jwt = require("jsonwebtoken") const env = require("../../../environment") +const { cloneDeep } = require("lodash/fp") const TEST_CLIENT_ID = "test-client-id" @@ -37,29 +38,28 @@ exports.defaultHeaders = appId => { return headers } -exports.createTable = async (request, appId, table) => { - if (table != null && table._id) { - delete table._id - } - table = table || { - name: "TestTable", - type: "table", - key: "name", - schema: { - name: { +exports.BASE_TABLE = { + name: "TestTable", + type: "table", + key: "name", + schema: { + name: { + type: "string", + constraints: { type: "string", - constraints: { - type: "string", - }, - }, - description: { - type: "string", - constraints: { - type: "string", - }, }, }, - } + description: { + type: "string", + constraints: { + type: "string", + }, + }, + }, +} + +exports.createTable = async (request, appId, table) => { + table = table || exports.BASE_TABLE const res = await request .post(`/api/tables`) @@ -68,6 +68,25 @@ exports.createTable = async (request, appId, table) => { return res.body } +exports.createLinkedTable = async (request, appId) => { + // get the ID to link to + const table = await exports.createTable(request, appId) + table.schema.link = { + type: "link", + fieldName: "link", + tableId: table._id, + } + return exports.createTable(request, appId, table) +} + +exports.createAttachmentTable = async (request, appId) => { + const table = await exports.createTable(request, appId) + table.schema.attachment = { + type: "attachment", + } + return exports.createTable(request, appId, table) +} + exports.getAllFromTable = async (request, appId, tableId) => { const res = await request .get(`/api/${tableId}/rows`) diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.js index 0698250050..47977ed207 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.js @@ -3,7 +3,11 @@ const { createTable, supertest, defaultHeaders, + createLinkedTable, + createAttachmentTable, } = require("./couchTestUtils"); +const { enrichRows } = require("../../../utilities") +const env = require("../../../environment") describe("/rows", () => { let request @@ -270,4 +274,44 @@ describe("/rows", () => { }) }) + + describe("enrich row unit test", () => { + it("should allow enriching some linked rows", async () => { + const table = await createLinkedTable(request, appId) + const firstRow = (await createRow({ + name: "Test Contact", + description: "original description", + tableId: table._id + })).body + const secondRow = (await createRow({ + name: "Test 2", + description: "og desc", + link: [firstRow._id], + tableId: table._id, + })).body + const enriched = await enrichRows(appId, table, [secondRow]) + expect(enriched[0].link.length).toBe(1) + expect(enriched[0].link[0]).toBe(firstRow._id) + }) + }) + + it("should allow enriching attachment rows", async () => { + const table = await createAttachmentTable(request, appId) + const row = (await createRow({ + name: "test", + description: "test", + attachment: [{ + url: "/test/thing", + }], + tableId: table._id, + })).body + // the environment needs configured for this + env.CLOUD = 1 + env.SELF_HOSTED = 1 + const enriched = await enrichRows(appId, table, [row]) + expect(enriched[0].attachment[0].url).toBe(`/app-assets/assets/${appId}/test/thing`) + // remove env config + env.CLOUD = undefined + env.SELF_HOSTED = undefined + }) }) \ No newline at end of file diff --git a/packages/server/src/db/linkedRows/LinkController.js b/packages/server/src/db/linkedRows/LinkController.js index d6116b76f6..061a9ac1ae 100644 --- a/packages/server/src/db/linkedRows/LinkController.js +++ b/packages/server/src/db/linkedRows/LinkController.js @@ -252,7 +252,11 @@ class LinkController { tableId: table._id, fieldName: fieldName, } - await this._db.put(linkedTable) + const response = await this._db.put(linkedTable) + // special case for when linking back to self, make sure rev updated + if (linkedTable._id === table._id) { + table._rev = response.rev + } } } return table diff --git a/packages/server/src/db/linkedRows/linkUtils.js b/packages/server/src/db/linkedRows/linkUtils.js index 5f9dca4088..6549f9b61b 100644 --- a/packages/server/src/db/linkedRows/linkUtils.js +++ b/packages/server/src/db/linkedRows/linkUtils.js @@ -31,11 +31,14 @@ exports.createLinkView = async appId => { thisId: doc1.rowId, fieldName: doc1.fieldName, }) - emit([doc2.tableId, doc2.rowId], { - id: doc1.rowId, - thisId: doc2.rowId, - fieldName: doc2.fieldName, - }) + // if linking to same table can't emit twice + if (doc1.tableId !== doc2.tableId) { + emit([doc2.tableId, doc2.rowId], { + id: doc1.rowId, + thisId: doc2.rowId, + fieldName: doc2.fieldName, + }) + } } }.toString(), } diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 9da2937a4d..31cc74b5e6 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -3,6 +3,8 @@ const { DocumentTypes, SEPARATOR } = require("../db/utils") const fs = require("fs") const { cloneDeep } = require("lodash/fp") const CouchDB = require("../db") +const { OBJ_STORE_DIRECTORY } = require("../constants") +const linkRows = require("../db/linkedRows") const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -211,3 +213,34 @@ exports.getAllApps = async () => { .map(({ value }) => value) } } + +/** + * This function "enriches" the input rows with anything they are supposed to contain, for example + * link records or attachment links. + * @param {string} appId the ID of the application for which rows are being enriched. + * @param {object} table the table from which these rows came from originally, this is used to determine + * the schema of the rows and then enrich. + * @param {object[]} rows the rows which are to be enriched. + * @returns {object[]} the enriched rows will be returned. + */ +exports.enrichRows = async (appId, table, rows) => { + // attach any linked row information + const enriched = await linkRows.attachLinkInfo(appId, rows) + // update the attachments URL depending on hosting + if (env.CLOUD && env.SELF_HOSTED) { + for (let [property, column] of Object.entries(table.schema)) { + if (column.type === "attachment") { + for (let row of enriched) { + if (row[property] == null || row[property].length === 0) { + continue + } + row[property].forEach(attachment => { + attachment.url = `${OBJ_STORE_DIRECTORY}/${appId}/${attachment.url}` + attachment.url = attachment.url.replace("//", "/") + }) + } + } + } + } + return enriched +} From fc179ed78ba6acff4b72804ffea5eef33a676a94 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Feb 2021 11:47:20 +0000 Subject: [PATCH 3/5] Linting. --- packages/server/src/api/routes/tests/couchTestUtils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/couchTestUtils.js b/packages/server/src/api/routes/tests/couchTestUtils.js index adf75555ff..9f02047b4b 100644 --- a/packages/server/src/api/routes/tests/couchTestUtils.js +++ b/packages/server/src/api/routes/tests/couchTestUtils.js @@ -4,7 +4,6 @@ const { BUILTIN_ROLE_IDS } = require("../../../utilities/security/roles") const packageJson = require("../../../../package") const jwt = require("jsonwebtoken") const env = require("../../../environment") -const { cloneDeep } = require("lodash/fp") const TEST_CLIENT_ID = "test-client-id" From 4d30e6a45a88a70f1a43e469391ac41de9bd3a3e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Feb 2021 13:14:32 +0000 Subject: [PATCH 4/5] Fixing test failure due to not deleting the ID attached to table in couchTest Utils. --- packages/server/src/api/routes/tests/couchTestUtils.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/couchTestUtils.js b/packages/server/src/api/routes/tests/couchTestUtils.js index 9f02047b4b..59a9b6a7c2 100644 --- a/packages/server/src/api/routes/tests/couchTestUtils.js +++ b/packages/server/src/api/routes/tests/couchTestUtils.js @@ -57,7 +57,10 @@ exports.BASE_TABLE = { }, } -exports.createTable = async (request, appId, table) => { +exports.createTable = async (request, appId, table, removeId = true) => { + if (removeId && table != null && table._id) { + delete table._id + } table = table || exports.BASE_TABLE const res = await request @@ -75,7 +78,7 @@ exports.createLinkedTable = async (request, appId) => { fieldName: "link", tableId: table._id, } - return exports.createTable(request, appId, table) + return exports.createTable(request, appId, table, false) } exports.createAttachmentTable = async (request, appId) => { @@ -83,7 +86,7 @@ exports.createAttachmentTable = async (request, appId) => { table.schema.attachment = { type: "attachment", } - return exports.createTable(request, appId, table) + return exports.createTable(request, appId, table, false) } exports.getAllFromTable = async (request, appId, tableId) => { From ca66433aa206dc90589bb0213fb9a9772851cda5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 2 Feb 2021 14:55:52 +0000 Subject: [PATCH 5/5] fixing cypress test. --- packages/server/src/api/controllers/row.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 6a57d54bea..1c038f0194 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -190,7 +190,14 @@ exports.fetchView = async function(ctx) { if (!calculation) { response.rows = response.rows.map(row => row.doc) - const table = await db.get(ctx.params.tableId) + let table + try { + table = await db.get(ctx.params.tableId) + } catch (err) { + table = { + schema: {}, + } + } ctx.body = await enrichRows(appId, table, response.rows) }