From 9c7fc41913b12da2d4a08283b65e58e0ef3971a5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 16 Jan 2025 09:41:51 +0000 Subject: [PATCH 1/9] Convert last few .js files in backend-core to .ts --- .../backend-core/src/db/couch/connections.ts | 4 ++-- .../db/tests/{pouch.spec.js => pouch.spec.ts} | 3 ++- .../users/{users.spec.js => users.spec.ts} | 21 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename packages/backend-core/src/db/tests/{pouch.spec.js => pouch.spec.ts} (98%) rename packages/backend-core/tests/core/users/{users.spec.js => users.spec.ts} (67%) diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 5c7d7ec81d..9692b095a8 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -1,6 +1,6 @@ import env from "../../environment" -export const getCouchInfo = (connection?: string) => { +export const getCouchInfo = (connection?: string | null) => { // clean out any auth credentials const urlInfo = getUrlInfo(connection) let username @@ -45,7 +45,7 @@ export const getCouchInfo = (connection?: string) => { } } -export const getUrlInfo = (url = env.COUCH_DB_URL) => { +export const getUrlInfo = (url: string | null = env.COUCH_DB_URL) => { let cleanUrl, username, password, host if (url) { // Ensure the URL starts with a protocol diff --git a/packages/backend-core/src/db/tests/pouch.spec.js b/packages/backend-core/src/db/tests/pouch.spec.ts similarity index 98% rename from packages/backend-core/src/db/tests/pouch.spec.js rename to packages/backend-core/src/db/tests/pouch.spec.ts index f0abc82240..21632cff88 100644 --- a/packages/backend-core/src/db/tests/pouch.spec.js +++ b/packages/backend-core/src/db/tests/pouch.spec.ts @@ -1,5 +1,6 @@ require("../../../tests") -const getUrlInfo = require("../couch").getUrlInfo + +import { getUrlInfo } from "../couch" describe("pouch", () => { describe("Couch DB URL parsing", () => { diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.ts similarity index 67% rename from packages/backend-core/tests/core/users/users.spec.js rename to packages/backend-core/tests/core/users/users.spec.ts index dde0d87fb7..b14f553266 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.ts @@ -1,17 +1,17 @@ -const _ = require("lodash/fp") -const { structures } = require("../../../tests") +import { range } from "lodash/fp" +import { structures } from "../.." jest.mock("../../../src/context") jest.mock("../../../src/db") -const context = require("../../../src/context") -const db = require("../../../src/db") +import * as context from "../../../src/context" +import * as db from "../../../src/db" -const { getCreatorCount } = require("../../../src/users/users") +import { getCreatorCount } from "../../../src/users/users" describe("Users", () => { - let getGlobalDBMock - let paginationMock + let getGlobalDBMock: jest.SpyInstance + let paginationMock: jest.SpyInstance beforeEach(() => { jest.resetAllMocks() @@ -22,11 +22,10 @@ describe("Users", () => { jest.spyOn(db, "getGlobalUserParams") }) - it("Retrieves the number of creators", async () => { - const getUsers = (offset, limit, creators = false) => { - const range = _.range(offset, limit) + it("retrieves the number of creators", async () => { + const getUsers = (offset: number, limit: number, creators = false) => { const opts = creators ? { builder: { global: true } } : undefined - return range.map(() => structures.users.user(opts)) + return range(offset, limit).map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) From 7c8bef3da76532d4636248cce716f94dd0b38c89 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 14:10:00 +0000 Subject: [PATCH 2/9] fix column names with spaces for relationship columns --- packages/backend-core/src/sql/sql.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5a6907faa0..eb103d3e2e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1173,7 +1173,13 @@ class InternalBuilder { } if (this.isAggregateField(key)) { - query = query.orderBy(key, direction, nulls) + // query = query.orderBy(this.quotedIdentifier(key), direction, nulls) + // query = query.orderBy(this.rawQuotedIdentifier(key), direction, nulls) + query = query.orderByRaw(`?? ??`, [ + this.rawQuotedIdentifier(key), + this.knex.raw(direction), + this.knex.raw(nulls as string), + ]) } else { let composite = `${aliased}.${key}` if (this.client === SqlClient.ORACLE) { @@ -1183,7 +1189,11 @@ class InternalBuilder { this.knex.raw(nulls as string), ]) } else { - query = query.orderBy(composite, direction, nulls) + query = query.orderByRaw(`?? ??`, [ + this.rawQuotedIdentifier(composite), + this.knex.raw(direction), + ]) + // query = query.orderBy(this.quotedIdentifier(key), direction, nulls) } } } @@ -1344,14 +1354,16 @@ class InternalBuilder { // add the correlation to the overall query subQuery = subQuery.where( - correlatedTo, + this.rawQuotedIdentifier(correlatedTo), "=", this.rawQuotedIdentifier(correlatedFrom) ) const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => { subQuery = subQuery - .select(relationshipFields) + .select( + relationshipFields.map(field => this.rawQuotedIdentifier(field)) + ) .limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(select).from({ From df2f6cf257b4a1c9013b8be8683157daed5208e9 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 14:13:48 +0000 Subject: [PATCH 3/9] re-added nulls --- packages/backend-core/src/sql/sql.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index eb103d3e2e..04b1309534 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1167,15 +1167,14 @@ class InternalBuilder { let nulls: "first" | "last" | undefined = undefined if ( this.client === SqlClient.POSTGRES || - this.client === SqlClient.ORACLE + this.client === SqlClient.ORACLE || + this.client === SqlClient.SQL_LITE ) { nulls = value.direction === SortOrder.ASCENDING ? "first" : "last" } if (this.isAggregateField(key)) { - // query = query.orderBy(this.quotedIdentifier(key), direction, nulls) - // query = query.orderBy(this.rawQuotedIdentifier(key), direction, nulls) - query = query.orderByRaw(`?? ??`, [ + query = query.orderByRaw(`?? ?? nulls ??`, [ this.rawQuotedIdentifier(key), this.knex.raw(direction), this.knex.raw(nulls as string), @@ -1189,11 +1188,11 @@ class InternalBuilder { this.knex.raw(nulls as string), ]) } else { - query = query.orderByRaw(`?? ??`, [ + query = query.orderByRaw(`?? ?? nulls ??`, [ this.rawQuotedIdentifier(composite), this.knex.raw(direction), + this.knex.raw(nulls as string) ]) - // query = query.orderBy(this.quotedIdentifier(key), direction, nulls) } } } From c077f7e84cf5e77ba447e7824e5f8483f2756f73 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 14:20:26 +0000 Subject: [PATCH 4/9] lint --- packages/backend-core/src/sql/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 04b1309534..6dc8963323 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1191,7 +1191,7 @@ class InternalBuilder { query = query.orderByRaw(`?? ?? nulls ??`, [ this.rawQuotedIdentifier(composite), this.knex.raw(direction), - this.knex.raw(nulls as string) + this.knex.raw(nulls as string), ]) } } From 3bd56bdca7be821b81d13fca1059b9fb206a6e6c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 14:29:49 +0000 Subject: [PATCH 5/9] remove nulls for raw and rely on defaults --- packages/backend-core/src/sql/sql.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 6dc8963323..fb7cfdec1b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1167,17 +1167,15 @@ class InternalBuilder { let nulls: "first" | "last" | undefined = undefined if ( this.client === SqlClient.POSTGRES || - this.client === SqlClient.ORACLE || - this.client === SqlClient.SQL_LITE + this.client === SqlClient.ORACLE ) { nulls = value.direction === SortOrder.ASCENDING ? "first" : "last" } if (this.isAggregateField(key)) { - query = query.orderByRaw(`?? ?? nulls ??`, [ + query = query.orderByRaw(`?? ??`, [ this.rawQuotedIdentifier(key), this.knex.raw(direction), - this.knex.raw(nulls as string), ]) } else { let composite = `${aliased}.${key}` @@ -1188,10 +1186,9 @@ class InternalBuilder { this.knex.raw(nulls as string), ]) } else { - query = query.orderByRaw(`?? ?? nulls ??`, [ + query = query.orderByRaw(`?? ??`, [ this.rawQuotedIdentifier(composite), this.knex.raw(direction), - this.knex.raw(nulls as string), ]) } } From e4f7fa7fadc7ebf94ff21b2cc68186349745e210 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 15:07:32 +0000 Subject: [PATCH 6/9] more consistent null handling --- packages/backend-core/src/sql/sql.ts | 17 +++----- .../server/src/api/routes/tests/row.spec.ts | 43 +++++++++++++++++++ 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index fb7cfdec1b..b91a331740 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1162,20 +1162,14 @@ class InternalBuilder { const direction = value.direction === SortOrder.ASCENDING ? "asc" : "desc" - // TODO: figure out a way to remove this conditional, not relying on - // the defaults of each datastore. - let nulls: "first" | "last" | undefined = undefined - if ( - this.client === SqlClient.POSTGRES || - this.client === SqlClient.ORACLE - ) { - nulls = value.direction === SortOrder.ASCENDING ? "first" : "last" - } + let nulls: "first" | "last" = + value.direction === SortOrder.ASCENDING ? "first" : "last" if (this.isAggregateField(key)) { - query = query.orderByRaw(`?? ??`, [ + query = query.orderByRaw(`?? ?? nulls ??`, [ this.rawQuotedIdentifier(key), this.knex.raw(direction), + this.knex.raw(nulls as string), ]) } else { let composite = `${aliased}.${key}` @@ -1186,9 +1180,10 @@ class InternalBuilder { this.knex.raw(nulls as string), ]) } else { - query = query.orderByRaw(`?? ??`, [ + query = query.orderByRaw(`?? ?? nulls ??`, [ this.rawQuotedIdentifier(composite), this.knex.raw(direction), + this.knex.raw(nulls as string), ]) } } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index db5fcbaebb..101b9928f4 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3650,6 +3650,49 @@ if (descriptions.length) { }) }) + describe("Fields with spaces", () => { + let table: Table + let otherTable: Table + let relatedRow: Row, mainRow: Row + + beforeAll(async () => { + otherTable = await config.api.table.save(defaultTable()) + table = await config.api.table.save( + saveTableRequest({ + schema: { + links: { + name: "links", + fieldName: "links", + type: FieldType.LINK, + tableId: otherTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + }, + "nameWithSpace ": { + name: "nameWithSpace ", + type: FieldType.STRING, + }, + }, + }) + ) + relatedRow = await config.api.row.save(otherTable._id!, { + name: generator.word(), + description: generator.paragraph(), + }) + mainRow = await config.api.row.save(table._id!, { + "nameWithSpace ": generator.word(), + tableId: table._id!, + links: [relatedRow._id], + }) + }) + + it("Successfully returns rows that have spaces in their field names", async () => { + const { rows } = await config.api.row.search(table._id!) + expect(rows.length).toBe(1) + const row = rows[0] + expect(row["nameWithSpace "]).toBeDefined() + }) + }) + if (!isInternal && !isOracle) { describe("bigint ids", () => { let table1: Table, table2: Table From c69e604eaf6d91189d778be6463b79426fb07f2c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 15:17:40 +0000 Subject: [PATCH 7/9] lint --- .../server/src/api/routes/tests/row.spec.ts | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 101b9928f4..576f0bb663 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3650,48 +3650,50 @@ if (descriptions.length) { }) }) - describe("Fields with spaces", () => { - let table: Table - let otherTable: Table - let relatedRow: Row, mainRow: Row + if (isInternal || isMSSQL) { + describe("Fields with spaces", () => { + let table: Table + let otherTable: Table + let relatedRow: Row - beforeAll(async () => { - otherTable = await config.api.table.save(defaultTable()) - table = await config.api.table.save( - saveTableRequest({ - schema: { - links: { - name: "links", - fieldName: "links", - type: FieldType.LINK, - tableId: otherTable._id!, - relationshipType: RelationshipType.ONE_TO_MANY, + beforeAll(async () => { + otherTable = await config.api.table.save(defaultTable()) + table = await config.api.table.save( + saveTableRequest({ + schema: { + links: { + name: "links", + fieldName: "links", + type: FieldType.LINK, + tableId: otherTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + }, + "nameWithSpace ": { + name: "nameWithSpace ", + type: FieldType.STRING, + }, }, - "nameWithSpace ": { - name: "nameWithSpace ", - type: FieldType.STRING, - }, - }, + }) + ) + relatedRow = await config.api.row.save(otherTable._id!, { + name: generator.word(), + description: generator.paragraph(), + }) + await config.api.row.save(table._id!, { + "nameWithSpace ": generator.word(), + tableId: table._id!, + links: [relatedRow._id], }) - ) - relatedRow = await config.api.row.save(otherTable._id!, { - name: generator.word(), - description: generator.paragraph(), }) - mainRow = await config.api.row.save(table._id!, { - "nameWithSpace ": generator.word(), - tableId: table._id!, - links: [relatedRow._id], - }) - }) - it("Successfully returns rows that have spaces in their field names", async () => { - const { rows } = await config.api.row.search(table._id!) - expect(rows.length).toBe(1) - const row = rows[0] - expect(row["nameWithSpace "]).toBeDefined() + it("Successfully returns rows that have spaces in their field names", async () => { + const { rows } = await config.api.row.search(table._id!) + expect(rows.length).toBe(1) + const row = rows[0] + expect(row["nameWithSpace "]).toBeDefined() + }) }) - }) + } if (!isInternal && !isOracle) { describe("bigint ids", () => { From d610317ef55c9a371779433c870bbd84d7511bbb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 16 Jan 2025 16:41:00 +0000 Subject: [PATCH 8/9] fix tests --- packages/backend-core/src/sql/sql.ts | 43 ++++++++++--------- .../src/integrations/microsoftSqlServer.ts | 1 + 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b91a331740..334f1efdd4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1162,31 +1162,32 @@ class InternalBuilder { const direction = value.direction === SortOrder.ASCENDING ? "asc" : "desc" - let nulls: "first" | "last" = - value.direction === SortOrder.ASCENDING ? "first" : "last" + // TODO: figure out a way to remove this conditional, not relying on + // the defaults of each datastore. + let nulls: "first" | "last" | undefined = undefined + if ( + this.client === SqlClient.POSTGRES || + this.client === SqlClient.ORACLE + ) { + nulls = value.direction === SortOrder.ASCENDING ? "first" : "last" + } + + const composite = `${aliased}.${key}` + let identifier if (this.isAggregateField(key)) { - query = query.orderByRaw(`?? ?? nulls ??`, [ - this.rawQuotedIdentifier(key), - this.knex.raw(direction), - this.knex.raw(nulls as string), - ]) + identifier = this.rawQuotedIdentifier(key) + } else if (this.client === SqlClient.ORACLE) { + identifier = this.convertClobs(composite) } else { - let composite = `${aliased}.${key}` - if (this.client === SqlClient.ORACLE) { - query = query.orderByRaw(`?? ?? nulls ??`, [ - this.convertClobs(composite), - this.knex.raw(direction), - this.knex.raw(nulls as string), - ]) - } else { - query = query.orderByRaw(`?? ?? nulls ??`, [ - this.rawQuotedIdentifier(composite), - this.knex.raw(direction), - this.knex.raw(nulls as string), - ]) - } + identifier = this.rawQuotedIdentifier(composite) } + + query = query.orderByRaw(`?? ?? ${nulls ? "nulls ??" : ""}`, [ + identifier, + this.knex.raw(direction), + ...(nulls ? [this.knex.raw(nulls as string)] : []), + ]) } } diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 477813239b..8548d57f15 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -276,6 +276,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { encrypt, enableArithAbort: true, requestTimeout: env.QUERY_THREAD_TIMEOUT, + connectTimeout: env.QUERY_THREAD_TIMEOUT, }, } if (encrypt) { From 709b3fb317ed50515ec73ad391167888674a0ced Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 16 Jan 2025 17:05:35 +0000 Subject: [PATCH 9/9] Bump version to 3.2.45 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 8d7179dbae..c02b221ec8 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.44", + "version": "3.2.45", "npmClient": "yarn", "concurrency": 20, "command": {