From dd78399fc14c36b53cbeb811a6cbca5d0c6aab9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 23 May 2024 17:56:48 +0100 Subject: [PATCH 1/8] Changing how content processing works for responses from REST API - try and reduce the chance of an error in the case of malformed data being returned. --- packages/server/src/integrations/rest.ts | 34 +++++++++++++------ .../server/src/integrations/utils/utils.ts | 5 ++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 7104c9daca..987f58423a 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -148,6 +148,10 @@ class RestIntegration implements IntegrationBase { response.headers, { downloadImages: this.config.downloadImages } ) + let contentLength = response.headers.get("content-length") + if (!contentLength && raw) { + contentLength = Buffer.byteLength(raw, "utf8").toString() + } if ( contentDisposition.includes("filename") || contentDisposition.includes("attachment") || @@ -156,36 +160,44 @@ class RestIntegration implements IntegrationBase { filename = path.basename(parse(contentDisposition).parameters?.filename) || "" } + + let triedParsing: boolean = false, + responseTxt: string | undefined + const hasContent = contentLength && parseInt(contentLength) > 0 try { if (filename) { return handleFileResponse(response, filename, this.startTimeMs) } else { + responseTxt = hasContent ? await response.text() : "" if (response.status === 204) { data = [] raw = "" - } else if (contentType.includes("application/json")) { - data = await response.json() - raw = JSON.stringify(data) + } else if (hasContent && contentType.includes("application/json")) { + triedParsing = true + data = JSON.parse(responseTxt) + raw = responseTxt } else if ( - contentType.includes("text/xml") || + (hasContent && contentType.includes("text/xml")) || contentType.includes("application/xml") ) { - let xmlResponse = await handleXml(response) + triedParsing = true + let xmlResponse = await handleXml(responseTxt) data = xmlResponse.data raw = xmlResponse.rawXml } else { - data = await response.text() + data = responseTxt raw = data as string } } } catch (err) { - throw `Failed to parse response body: ${err}` + if (triedParsing) { + data = responseTxt + raw = data as string + } else { + throw new Error(`Failed to parse response body: ${err}`) + } } - let contentLength = response.headers.get("content-length") - if (!contentLength && raw) { - contentLength = Buffer.byteLength(raw, "utf8").toString() - } const size = formatBytes(contentLength || "0") const time = `${Math.round(performance.now() - this.startTimeMs)}ms` headers = response.headers.raw() diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 5bc90bc295..069bd750ae 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -485,9 +485,8 @@ export function isValidFilter(value: any) { return value != null && value !== "" } -export async function handleXml(response: any) { - let data, - rawXml = await response.text() +export async function handleXml(rawXml: string) { + let data data = (await xmlParser(rawXml, { explicitArray: false, From 62c407d846c5b0073b955143bc60c884a5032722 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 24 May 2024 12:15:28 +0100 Subject: [PATCH 2/8] Updating test cases. --- packages/server/src/integrations/rest.ts | 2 +- .../src/integrations/tests/rest.spec.ts | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 987f58423a..57e4217722 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -168,7 +168,7 @@ class RestIntegration implements IntegrationBase { if (filename) { return handleFileResponse(response, filename, this.startTimeMs) } else { - responseTxt = hasContent ? await response.text() : "" + responseTxt = hasContent && response.text ? await response.text() : "" if (response.status === 204) { data = [] raw = "" diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index 144aefa576..be392f0a6f 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -1,19 +1,27 @@ jest.mock("node-fetch", () => { + const obj = { + my_next_cursor: 123, + } + const str = JSON.stringify(obj) return jest.fn(() => ({ headers: { raw: () => { - return { "content-type": ["application/json"] } + return { + "content-type": ["application/json"], + "content-length": str.length, + } }, get: (name: string) => { - if (name.toLowerCase() === "content-type") { + const lcName = name.toLowerCase() + if (lcName === "content-type") { return ["application/json"] + } else if (lcName === "content-length") { + return str.length } }, }, - json: jest.fn(() => ({ - my_next_cursor: 123, - })), - text: jest.fn(), + json: jest.fn(() => obj), + text: jest.fn(() => str), })) }) @@ -231,7 +239,8 @@ describe("REST Integration", () => { } it("should be able to parse JSON response", async () => { - const input = buildInput({ a: 1 }, null, "application/json") + const obj = { a: 1 } + const input = buildInput(obj, JSON.stringify(obj), "application/json") const output = await config.integration.parseResponse(input) expect(output.data).toEqual({ a: 1 }) expect(output.info.code).toEqual(200) @@ -261,7 +270,7 @@ describe("REST Integration", () => { test.each([...contentTypes, undefined])( "should not throw an error on 204 no content", async contentType => { - const input = buildInput(undefined, null, contentType, 204) + const input = buildInput(undefined, "", contentType, 204) const output = await config.integration.parseResponse(input) expect(output.data).toEqual([]) expect(output.extra.raw).toEqual("") From 6e06b322bfc16ef65921e686ac1aea8e7a115028 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 May 2024 13:21:04 +0200 Subject: [PATCH 3/8] Point @budibase/types types to dist --- packages/types/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/package.json b/packages/types/package.json index 67b923ed54..f4c7b13344 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "description": "Budibase types", "main": "dist/index.js", - "types": "src/index.ts", + "types": "dist/index.d.ts", "author": "Budibase", "license": "GPL-3.0", "scripts": { From ae26f66cf91e8579b27a3aa3e5c354f19ac7356f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 24 May 2024 12:40:58 +0100 Subject: [PATCH 4/8] Fixing a test case. --- .../server/src/api/routes/tests/queries/rest.spec.ts | 10 ++++------ packages/server/src/integrations/rest.ts | 6 ++++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/rest.spec.ts b/packages/server/src/api/routes/tests/queries/rest.spec.ts index 5c41583244..1d5483017b 100644 --- a/packages/server/src/api/routes/tests/queries/rest.spec.ts +++ b/packages/server/src/api/routes/tests/queries/rest.spec.ts @@ -64,13 +64,11 @@ describe("rest", () => { cached = await getCachedVariable(basedOnQuery._id!, "foo") expect(cached).toBeNull() - nock("http://one.example.com") - .get("/") - .reply(200, [{ name: "one" }]) + const body1 = [{ name: "one" }] + const body2 = [{ name: "two" }] + nock("http://one.example.com").get("/").reply(200, body1) nock("http://two.example.com").get("/?test=one").reply(500) - nock("http://two.example.com") - .get("/?test=one") - .reply(200, [{ name: "two" }]) + nock("http://two.example.com").get("/?test=one").reply(200, body2) const res = await config.api.query.preview({ datasourceId: datasource._id!, diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 57e4217722..451c319aa9 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -163,12 +163,14 @@ class RestIntegration implements IntegrationBase { let triedParsing: boolean = false, responseTxt: string | undefined - const hasContent = contentLength && parseInt(contentLength) > 0 try { if (filename) { return handleFileResponse(response, filename, this.startTimeMs) } else { - responseTxt = hasContent && response.text ? await response.text() : "" + responseTxt = response.text ? await response.text() : "" + const hasContent = + (contentLength && parseInt(contentLength) > 0) || + responseTxt.length > 0 if (response.status === 204) { data = [] raw = "" From 378f1215a06b58c22e3cba9beda0b0e6fafb9b4d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 May 2024 13:45:00 +0200 Subject: [PATCH 5/8] Add nx dependency --- packages/shared-core/package.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index c024d1b819..c982fb852f 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -22,5 +22,19 @@ "devDependencies": { "rimraf": "3.0.2", "typescript": "5.2.2" + }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/types" + ], + "target": "build" + } + ] + } + } } } From 863881e45acef69418f7856f5d8dd65d6933af17 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 24 May 2024 13:47:34 +0200 Subject: [PATCH 6/8] Add nx dependency --- packages/shared-core/package.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index c982fb852f..3049afdb95 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -34,6 +34,16 @@ "target": "build" } ] + }, + "dev": { + "dependsOn": [ + { + "projects": [ + "@budibase/types" + ], + "target": "build" + } + ] } } } From 1f2b3abb467edf9677c747c53c46049b8b0e66d7 Mon Sep 17 00:00:00 2001 From: andz-bb Date: Fri, 24 May 2024 13:46:46 +0100 Subject: [PATCH 7/8] change "Links" to "Relationships" in table data source selection --- .../settings/controls/DataSourceSelect/DataSourceSelect.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index bf0aa75bed..e247e4cc88 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -309,7 +309,7 @@ {#if links?.length} Date: Fri, 24 May 2024 16:26:13 +0100 Subject: [PATCH 8/8] Updating the package.json to publish CouchDB 3.3.3 as that is what is in the image. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 98524e0ee4..f9aef2f36b 100644 --- a/package.json +++ b/package.json @@ -74,8 +74,8 @@ "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single": "./scripts/build-single-image.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", - "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", - "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb", + "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 --push ./hosting/couchdb", + "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.3.3-sqs --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run --stream env:multi:enable",