From 5c01ba0095b790318dbc94394b5a201676d0ae99 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 14:48:56 +0000 Subject: [PATCH 01/12] Fix for issue with aliasing not quite working as expected when interacting with very old datasources, there is a flag 'isSQL' which was not set in old versions, this is now set when retrieving datasources to avoid issues with it being unset. --- .../src/sdk/app/datasources/datasources.ts | 18 ++++++++++++++---- packages/server/src/sdk/app/rows/utils.ts | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index fd0d291d91..d7c922412f 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -30,9 +30,15 @@ import { } from "../../../db/utils" import sdk from "../../index" import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets" +import { helpers } from "@budibase/shared-core" const ENV_VAR_PREFIX = "env." +function addDatasourceFlags(datasource: Datasource) { + datasource.isSQL = helpers.isSQL(datasource) + return datasource +} + export async function fetch(opts?: { enriched: boolean }): Promise { @@ -56,7 +62,7 @@ export async function fetch(opts?: { } as Datasource // Get external datasources - const datasources = ( + let datasources = ( await db.allDocs( getDatasourceParams(null, { include_docs: true, @@ -75,6 +81,7 @@ export async function fetch(opts?: { } } + datasources = datasources.map(datasource => addDatasourceFlags(datasource)) if (opts?.enriched) { const envVars = await getEnvironmentVariables() const promises = datasources.map(datasource => @@ -150,6 +157,7 @@ async function enrichDatasourceWithValues( } export async function enrich(datasource: Datasource) { + datasource = addDatasourceFlags(datasource) const { datasource: response } = await enrichDatasourceWithValues(datasource) return response } @@ -159,7 +167,8 @@ export async function get( opts?: { enriched: boolean } ): Promise { const appDb = context.getAppDB() - const datasource = await appDb.get(datasourceId) + let datasource = await appDb.get(datasourceId) + datasource = addDatasourceFlags(datasource) if (opts?.enriched) { return (await enrichDatasourceWithValues(datasource)).datasource } else { @@ -271,13 +280,14 @@ export function mergeConfigs(update: Datasource, old: Datasource) { export async function getExternalDatasources(): Promise { const db = context.getAppDB() - const externalDatasources = await db.allDocs( + let dsResponse = await db.allDocs( getDatasourcePlusParams(undefined, { include_docs: true, }) ) - return externalDatasources.rows.map(r => r.doc!) + const externalDatasources = dsResponse.rows.map(r => r.doc!) + return externalDatasources.map(datasource => addDatasourceFlags(datasource)) } export async function save( diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 6e3e25364e..8aa017d238 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,7 +14,7 @@ import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { isRelationshipColumn } from "../../../db/utils" -import { SqlClient } from "../../../integrations/utils" +import { SqlClient, isSQL } from "../../../integrations/utils" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -37,7 +37,7 @@ const SQL_CLIENT_SOURCE_MAP: Record = { } export function getSQLClient(datasource: Datasource): SqlClient { - if (!datasource.isSQL) { + if (!isSQL(datasource)) { throw new Error("Cannot get SQL Client for non-SQL datasource") } const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source] From 8103e5291ce88db520026c3c4c8905341af038ea Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 15:06:23 +0000 Subject: [PATCH 02/12] Fix test. --- packages/server/src/integration-test/postgres.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 7c14bc2b69..107c4ade1e 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -319,6 +319,7 @@ describe("postgres integrations", () => { }, plus: true, source: "POSTGRES", + isSQL: true, type: "datasource_plus", _id: expect.any(String), _rev: expect.any(String), From f82f6e7b3b8641d75939450f5fb24ea2e0068b2b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 15:07:28 +0000 Subject: [PATCH 03/12] Set on save, isSQL as well. --- packages/server/src/sdk/app/datasources/datasources.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index d7c922412f..336a94636b 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -300,11 +300,11 @@ export async function save( const fetchSchema = opts?.fetchSchema || false const tablesFilter = opts?.tablesFilter || [] - datasource = { + datasource = addDatasourceFlags({ _id: generateDatasourceID({ plus }), ...datasource, type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE, - } + }) let errors: Record = {} if (fetchSchema) { From e0805f7056241ae21a1c14262fb376983e9637ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 15:17:13 +0000 Subject: [PATCH 04/12] fix for mysql test as well. --- packages/server/src/integration-test/mysql.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/integration-test/mysql.spec.ts b/packages/server/src/integration-test/mysql.spec.ts index fac2bfcfeb..8804f87c28 100644 --- a/packages/server/src/integration-test/mysql.spec.ts +++ b/packages/server/src/integration-test/mysql.spec.ts @@ -106,6 +106,7 @@ describe("mysql integrations", () => { plus: true, source: "MYSQL", type: "datasource_plus", + isSQL: true, _id: expect.any(String), _rev: expect.any(String), createdAt: expect.any(String), From 4efec72b4db7269692f79a5975d22ddd516d7c77 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 16:11:26 +0000 Subject: [PATCH 05/12] Fixing an issue uncovered by tests, MS-SQL and MariaDB when columns are aliased don't always return an array, they may return a stringified version. Making the reference processor more hardy against this. --- .../utilities/rowProcessor/bbReferenceProcessor.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index c7b8998bad..a51ac32da7 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -68,8 +68,16 @@ export async function processOutputBBReferences( return value || undefined } - const ids = - typeof value === "string" ? value.split(",").filter(id => !!id) : value + let ids: string[] = [] + if (typeof value === "string") { + try { + ids = JSON.parse(value) + } catch (err) { + ids = value.split(",").filter(id => !!id) + } + } else if (Array.isArray(value)) { + ids = value + } switch (subtype) { case FieldSubtype.USER: From 6a0e46a0a685a53ff1aab54bd533288efd664d39 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 16:24:15 +0000 Subject: [PATCH 06/12] Proper fix, making sure that low level handling of JSON column types correctly handles aliasing. --- packages/server/src/integrations/base/sql.ts | 9 ++++++--- .../utilities/rowProcessor/bbReferenceProcessor.ts | 12 ++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 8342c45fd7..f5e7812861 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -705,10 +705,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!this._isJsonColumn(field)) { continue } - const fullName = `${table.name}.${name}` for (let row of results) { - if (typeof row[fullName] === "string") { - row[fullName] = JSON.parse(row[fullName]) + const columnNames = Object.keys(row) + const fullColumnName = columnNames.find( + column => column.includes(`.${name}`) || column === name + ) + if (fullColumnName && typeof row[fullColumnName] === "string") { + row[fullColumnName] = JSON.parse(row[fullColumnName]) } if (typeof row[name] === "string") { row[name] = JSON.parse(row[name]) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index a51ac32da7..c7b8998bad 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -68,16 +68,8 @@ export async function processOutputBBReferences( return value || undefined } - let ids: string[] = [] - if (typeof value === "string") { - try { - ids = JSON.parse(value) - } catch (err) { - ids = value.split(",").filter(id => !!id) - } - } else if (Array.isArray(value)) { - ids = value - } + const ids = + typeof value === "string" ? value.split(",").filter(id => !!id) : value switch (subtype) { case FieldSubtype.USER: From 8d7267691d7e9c49003af85c49ac2d902d01a19e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 16:28:25 +0000 Subject: [PATCH 07/12] Handling aliasing in column conversions. --- packages/server/src/integrations/base/sql.ts | 13 ++++++------- .../server/src/integrations/microsoftSqlServer.ts | 6 +++++- packages/server/src/integrations/mysql.ts | 6 +++++- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index f5e7812861..2093d455b9 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -699,19 +699,18 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { convertJsonStringColumns( table: Table, - results: Record[] + results: Record[], + aliases?: Record ): Record[] { for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } + const tableName = aliases?.[table.name] || table.name + const fullName = `${tableName}.${name}` for (let row of results) { - const columnNames = Object.keys(row) - const fullColumnName = columnNames.find( - column => column.includes(`.${name}`) || column === name - ) - if (fullColumnName && typeof row[fullColumnName] === "string") { - row[fullColumnName] = JSON.parse(row[fullColumnName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } if (typeof row[name] === "string") { row[name] = JSON.parse(row[name]) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index c79eb136ed..36fdae980e 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -506,7 +506,11 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { const queryFn = (query: any, op: string) => this.internalQuery(query, op) const processFn = (result: any) => { if (json?.meta?.table && result.recordset) { - return this.convertJsonStringColumns(json.meta.table, result.recordset) + return this.convertJsonStringColumns( + json.meta.table, + result.recordset, + json.tableAliases + ) } else if (result.recordset) { return result.recordset } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 9638afa8ea..f88162cf37 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -390,7 +390,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus { this.internalQuery(query, { connect: false, disableCoercion: true }) const processFn = (result: any) => { if (json?.meta?.table && Array.isArray(result)) { - return this.convertJsonStringColumns(json.meta.table, result) + return this.convertJsonStringColumns( + json.meta.table, + result, + json.tableAliases + ) } return result } From 74abee890252b788ba003c1bfc6f497bcb1e4ff6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 19 Mar 2024 16:52:16 +0000 Subject: [PATCH 08/12] Fixing snapshot test. --- .../src/api/routes/tests/__snapshots__/datasource.spec.ts.snap | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap index 8dc472173c..57d79db24b 100644 --- a/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap +++ b/packages/server/src/api/routes/tests/__snapshots__/datasource.spec.ts.snap @@ -81,6 +81,7 @@ exports[`/datasources fetch returns all the datasources from the server 1`] = ` { "config": {}, "createdAt": "2020-01-01T00:00:00.000Z", + "isSQL": true, "name": "Test", "source": "POSTGRES", "type": "datasource", From a2a0b96ce330c1b925a39eb521f92dfb9685f470 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Mar 2024 18:26:09 +0100 Subject: [PATCH 09/12] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 993b9f010f..50aaebde00 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 993b9f010f619b7f8b50c89105c645e3d874929e +Subproject commit 50aaebde002f8dd75dee2ad2c5e8036d6b3d9cc5 From 4389adbe79c9a622aec66de4a773f606427d2684 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Mar 2024 17:40:40 +0000 Subject: [PATCH 10/12] Bump version to 2.22.4 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 01e56982d5..9ace38aee5 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.22.3", + "version": "2.22.4", "npmClient": "yarn", "packages": [ "packages/*", From e5730b7e8401af219e0c1e8ef1986d1e550a6ec2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 19 Mar 2024 18:45:26 +0100 Subject: [PATCH 11/12] Fix check:types --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 50aaebde00..d68b4f40f8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 50aaebde002f8dd75dee2ad2c5e8036d6b3d9cc5 +Subproject commit d68b4f40f85dba3184da8b9d63ef2cd66d4a8ef2 From bfacb879a6b3e8e0511cdd85ffda32993a197f7c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 19 Mar 2024 18:51:02 +0000 Subject: [PATCH 12/12] Bump version to 2.22.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9ace38aee5..341efc0cad 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.22.4", + "version": "2.22.5", "npmClient": "yarn", "packages": [ "packages/*",