From dd468c293e66f2f87f0de4e9e14b609902bd121a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 20 Mar 2024 13:03:24 +0000 Subject: [PATCH 01/28] Remove worker aws-sdk, oauth, node-fetch mocks. --- .../worker/__mocks__/{oauth.ts => _oauth.ts} | 0 packages/worker/__mocks__/aws-sdk.ts | 15 -- packages/worker/__mocks__/node-fetch.ts | 24 --- packages/worker/package.json | 1 + .../src/api/routes/global/tests/auth.spec.ts | 92 +++++---- yarn.lock | 179 +++++++++++++++--- 6 files changed, 198 insertions(+), 113 deletions(-) rename packages/worker/__mocks__/{oauth.ts => _oauth.ts} (100%) delete mode 100644 packages/worker/__mocks__/aws-sdk.ts delete mode 100644 packages/worker/__mocks__/node-fetch.ts diff --git a/packages/worker/__mocks__/oauth.ts b/packages/worker/__mocks__/_oauth.ts similarity index 100% rename from packages/worker/__mocks__/oauth.ts rename to packages/worker/__mocks__/_oauth.ts diff --git a/packages/worker/__mocks__/aws-sdk.ts b/packages/worker/__mocks__/aws-sdk.ts deleted file mode 100644 index 7fac80faa9..0000000000 --- a/packages/worker/__mocks__/aws-sdk.ts +++ /dev/null @@ -1,15 +0,0 @@ -const mockS3 = { - headBucket: jest.fn().mockReturnThis(), - deleteObject: jest.fn().mockReturnThis(), - deleteObjects: jest.fn().mockReturnThis(), - createBucket: jest.fn().mockReturnThis(), - listObjects: jest.fn().mockReturnThis(), - promise: jest.fn().mockReturnThis(), - catch: jest.fn(), -} - -const AWS = { - S3: jest.fn(() => mockS3), -} - -export default AWS diff --git a/packages/worker/__mocks__/node-fetch.ts b/packages/worker/__mocks__/node-fetch.ts deleted file mode 100644 index 27c9b507a4..0000000000 --- a/packages/worker/__mocks__/node-fetch.ts +++ /dev/null @@ -1,24 +0,0 @@ -import fs from "fs" - -module FetchMock { - const fetch = jest.requireActual("node-fetch") - - const func = async (url: any, opts: any) => { - if (url.includes("http://someconfigurl")) { - return { - ok: true, - json: () => ({ - issuer: "test", - authorization_endpoint: "http://localhost/auth", - token_endpoint: "http://localhost/token", - userinfo_endpoint: "http://localhost/userinfo", - }), - } - } - return fetch(url, opts) - } - - func.Headers = fetch.Headers - - module.exports = func -} diff --git a/packages/worker/package.json b/packages/worker/package.json index e01a50d981..efdbe6aa12 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -86,6 +86,7 @@ "@types/supertest": "2.0.14", "@types/uuid": "8.3.4", "jest": "29.7.0", + "nock": "^13.5.4", "nodemon": "2.0.15", "rimraf": "3.0.2", "supertest": "6.3.3", diff --git a/packages/worker/src/api/routes/global/tests/auth.spec.ts b/packages/worker/src/api/routes/global/tests/auth.spec.ts index a81d5db435..852831837b 100644 --- a/packages/worker/src/api/routes/global/tests/auth.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auth.spec.ts @@ -13,6 +13,8 @@ import { events, constants } from "@budibase/backend-core" import { Response } from "superagent" import * as userSdk from "../../../../sdk/users" +import nock from "nock" +import * as jwt from "jsonwebtoken" function getAuthCookie(response: Response) { return response.headers["set-cookie"] @@ -277,45 +279,9 @@ describe("/api/global/auth", () => { }) }) - describe("init", () => { - describe("POST /api/global/auth/init", () => {}) - - describe("GET /api/global/auth/init", () => {}) - }) - - describe("datasource", () => { - // MULTI TENANT - - describe("GET /api/global/auth/:tenantId/datasource/:provider", () => {}) - - describe("GET /api/global/auth/:tenantId/datasource/:provider/callback", () => {}) - - // SINGLE TENANT - - describe("GET /api/global/auth/datasource/:provider/callback", () => {}) - }) - - describe("google", () => { - // MULTI TENANT - - describe("GET /api/global/auth/:tenantId/google", () => {}) - - describe("GET /api/global/auth/:tenantId/google/callback", () => {}) - - // SINGLE TENANT - - describe("GET /api/global/auth/google/callback", () => {}) - - describe("GET /api/admin/auth/google/callback", () => {}) - }) - describe("oidc", () => { - beforeEach(async () => { - jest.clearAllMocks() - mockGetWellKnownConfig() - - // see: __mocks__/oauth - // for associated mocking inside passport + afterEach(() => { + nock.cleanAll() }) const generateOidcConfig = async () => { @@ -324,21 +290,16 @@ describe("/api/global/auth", () => { return chosenConfig.uuid } - const mockGetWellKnownConfig = () => { - mocks.fetch.mockReturnValue({ - ok: true, - json: () => ({ + // MULTI TENANT + describe("GET /api/global/auth/:tenantId/oidc/configs/:configId", () => { + it("redirects to auth provider", async () => { + nock("http://someconfigurl").get("/").times(1).reply(200, { issuer: "test", authorization_endpoint: "http://localhost/auth", token_endpoint: "http://localhost/token", userinfo_endpoint: "http://localhost/userinfo", - }), - }) - } + }) - // MULTI TENANT - describe("GET /api/global/auth/:tenantId/oidc/configs/:configId", () => { - it("redirects to auth provider", async () => { const configId = await generateOidcConfig() const res = await config.api.configs.getOIDCConfig(configId) @@ -355,10 +316,43 @@ describe("/api/global/auth", () => { describe("GET /api/global/auth/:tenantId/oidc/callback", () => { it("logs in", async () => { + nock("http://someconfigurl").get("/").times(2).reply(200, { + issuer: "test", + authorization_endpoint: "http://localhost/auth", + token_endpoint: "http://localhost/token", + userinfo_endpoint: "http://localhost/userinfo", + }) + + const token = jwt.sign( + { + iss: "test", + sub: "sub", + aud: "clientId", + exp: Math.floor(Date.now() / 1000) + 60 * 60, + email: "oauth@example.com", + }, + "secret" + ) + + nock("http://localhost").post("/token").reply(200, { + access_token: "access", + refresh_token: "refresh", + id_token: token, + }) + + nock("http://localhost").get("/userinfo?schema=openid").reply(200, { + sub: "sub", + email: "oauth@example.com", + }) + const configId = await generateOidcConfig() const preAuthRes = await config.api.configs.getOIDCConfig(configId) - const res = await config.api.configs.OIDCCallback(configId, preAuthRes) + if (res.status > 399) { + throw new Error( + `OIDC callback failed with status ${res.status}: ${res.text}` + ) + } expect(events.auth.login).toHaveBeenCalledWith( "oidc", diff --git a/yarn.lock b/yarn.lock index 22c11453ee..5abdf295f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7179,7 +7179,37 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.21.4, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1, axios@^0.21.4: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^0.26.0: + version "0.26.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" + integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== + dependencies: + follow-redirects "^1.14.8" + +axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11169,6 +11199,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0, follow-redirects@^1.14.4, follow-redirects@^1.14.8: + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== + follow-redirects@^1.15.0: version "1.15.2" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" @@ -12310,7 +12345,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13260,11 +13300,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -15852,7 +15887,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16020,6 +16055,15 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== +nock@^13.5.4: + version "13.5.4" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.5.4.tgz#8918f0addc70a63736170fef7106a9721e0dc479" + integrity sha512-yAyTfdeNJGGBFxWdzSKCBYxs5FxLbCg5X5Q4ets974hcQzG1+qCxvIyOo4j2Ry6MUlhWVMX4OoYDefAIIwupjw== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + propagate "^2.0.0" + node-abi@^3.3.0: version "3.54.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.54.0.tgz#f6386f7548817acac6434c6cba02999c9aebcc69" @@ -16052,13 +16096,25 @@ node-duration@^1.0.4: resolved "https://registry.yarnpkg.com/node-duration/-/node-duration-1.0.4.tgz#3e94ecc0e473691c89c4560074503362071cecac" integrity sha512-eUXYNSY7DL53vqfTosggWkvyIW3bhAcqBDIlolgNYlZhianXTrCL50rlUJWD1eRqkIxMppXTfiFbp+9SjpPrgA== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17208,7 +17264,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18393,6 +18457,11 @@ promzard@^1.0.0: dependencies: read "^2.0.0" +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + proper-lockfile@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" @@ -18503,7 +18572,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19528,6 +19597,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.3.0.tgz#a5dbe77db3be05c9d1ee7785dbd3ea9de51593d0" + integrity sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19609,13 +19683,40 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@~2.3.1, semver@~7.0.0: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.1.2, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + +semver@~7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" + integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21218,7 +21319,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21228,6 +21329,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21696,14 +21805,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22474,10 +22575,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22487,6 +22611,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 4c55d958a1b6311d7cf4e3baf943774bfc7b9f14 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 20 Mar 2024 13:03:57 +0000 Subject: [PATCH 02/28] Actually remove oauth.ts this time. --- packages/worker/__mocks__/_oauth.ts | 57 ----------------------------- 1 file changed, 57 deletions(-) delete mode 100644 packages/worker/__mocks__/_oauth.ts diff --git a/packages/worker/__mocks__/_oauth.ts b/packages/worker/__mocks__/_oauth.ts deleted file mode 100644 index 8e8122a9e0..0000000000 --- a/packages/worker/__mocks__/_oauth.ts +++ /dev/null @@ -1,57 +0,0 @@ -import * as jwt from "jsonwebtoken" - -const mockOAuth2 = { - getOAuthAccessToken: (code: string, p: any, cb: any) => { - const err = null - const accessToken = "access_token" - const refreshToken = "refresh_token" - - const exp = new Date() - exp.setDate(exp.getDate() + 1) - - const iat = new Date() - iat.setDate(iat.getDate() - 1) - - const claims = { - iss: "test", - sub: "sub", - aud: "clientId", - exp: exp.getTime() / 1000, - iat: iat.getTime() / 1000, - email: "oauth@example.com", - } - - const idToken = jwt.sign(claims, "secret") - - const params = { - id_token: idToken, - } - return cb(err, accessToken, refreshToken, params) - }, - _request: ( - method: string, - url: string, - headers: any, - postBody: any, - accessToken: string, - cb: any - ) => { - const err = null - const body = { - sub: "sub", - user_id: "userId", - name: "OAuth", - family_name: "2", - given_name: "OAuth", - middle_name: "", - } - const res = {} - return cb(err, JSON.stringify(body), res) - }, -} - -const oauth = { - OAuth2: jest.fn(() => mockOAuth2), -} - -export = oauth From ff22db3d9f96b92b4e9808d261c892975f829f65 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 20 Mar 2024 15:42:06 +0000 Subject: [PATCH 03/28] Fold MongoDB unit tests into integration tests, delete MongoDB mocks. --- packages/server/__mocks__/mongodb.ts | 39 --- .../api/routes/tests/queries/mongodb.spec.ts | 135 +++++++- packages/server/src/integrations/mongodb.ts | 4 +- .../src/integrations/tests/mongo.spec.ts | 325 ------------------ .../server/src/tests/utilities/api/query.ts | 6 +- 5 files changed, 132 insertions(+), 377 deletions(-) delete mode 100644 packages/server/__mocks__/mongodb.ts delete mode 100644 packages/server/src/integrations/tests/mongo.spec.ts diff --git a/packages/server/__mocks__/mongodb.ts b/packages/server/__mocks__/mongodb.ts deleted file mode 100644 index 01b6e76fc4..0000000000 --- a/packages/server/__mocks__/mongodb.ts +++ /dev/null @@ -1,39 +0,0 @@ -module MongoMock { - const mongodb: any = {} - - mongodb.MongoClient = function () { - this.connect = jest.fn() - this.close = jest.fn() - this.insertOne = jest.fn() - this.insertMany = jest.fn(() => ({ toArray: () => [] })) - this.find = jest.fn(() => ({ toArray: () => [] })) - this.findOne = jest.fn() - this.findOneAndUpdate = jest.fn() - this.count = jest.fn() - this.deleteOne = jest.fn() - this.deleteMany = jest.fn(() => ({ toArray: () => [] })) - this.updateOne = jest.fn() - this.updateMany = jest.fn(() => ({ toArray: () => [] })) - - this.collection = jest.fn(() => ({ - insertOne: this.insertOne, - find: this.find, - insertMany: this.insertMany, - findOne: this.findOne, - findOneAndUpdate: this.findOneAndUpdate, - count: this.count, - deleteOne: this.deleteOne, - deleteMany: this.deleteMany, - updateOne: this.updateOne, - updateMany: this.updateMany, - })) - - this.db = () => ({ - collection: this.collection, - }) - } - - mongodb.ObjectId = jest.requireActual("mongodb").ObjectId - - module.exports = mongodb -} diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index b61f905bef..14ee46c535 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -36,27 +36,27 @@ describe("/queries", () => { return await config.api.query.create(combinedQuery) } - async function withClient( - callback: (client: MongoClient) => Promise - ): Promise { + async function withClient( + callback: (client: MongoClient) => Promise + ): Promise { const ds = await databaseTestProviders.mongodb.datasource() const client = new MongoClient(ds.config!.connectionString) await client.connect() try { - await callback(client) + return await callback(client) } finally { await client.close() } } - async function withCollection( - callback: (collection: Collection) => Promise - ): Promise { - await withClient(async client => { + async function withCollection( + callback: (collection: Collection) => Promise + ): Promise { + return await withClient(async client => { const db = client.db( (await databaseTestProviders.mongodb.datasource()).config!.db ) - await callback(db.collection(collection)) + return await callback(db.collection(collection)) }) } @@ -327,6 +327,42 @@ describe("/queries", () => { }) }) + it("should be able to updateOne by ObjectId", async () => { + const insertResult = await withCollection(c => c.insertOne({ name: "one" })) + const query = await createQuery({ + fields: { + json: { + filter: { _id: { $eq: `ObjectId("${insertResult.insertedId}")` } }, + update: { $set: { name: "newName" } }, + }, + extra: { + actionType: "updateOne", + }, + }, + queryVerb: "update", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + acknowledged: true, + matchedCount: 1, + modifiedCount: 1, + upsertedCount: 0, + upsertedId: null, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ name: { $eq: "newName" } }) + expect(doc).toEqual({ + _id: insertResult.insertedId, + name: "newName", + }) + }) + }) + it("should be able to delete all records", async () => { const query = await createQuery({ fields: { @@ -390,4 +426,85 @@ describe("/queries", () => { } }) }) + + it("should throw an error if the incorrect actionType is specified", async () => { + const verbs = ["read", "create", "update", "delete"] + for (const verb of verbs) { + const query = await createQuery({ + fields: { json: {}, extra: { actionType: "invalid" } }, + queryVerb: verb, + }) + await config.api.query.execute(query._id!, undefined, { status: 400 }) + } + }) + + it("should ignore extra brackets in query", async () => { + const query = await createQuery({ + fields: { + json: { foo: "te}st" }, + extra: { + actionType: "insertOne", + }, + }, + queryVerb: "create", + }) + + const result = await config.api.query.execute(query._id!) + expect(result.data).toEqual([ + { + acknowledged: true, + insertedId: expectValidId, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ foo: { $eq: "te}st" } }) + expect(doc).toEqual({ + _id: expectValidBsonObjectId, + foo: "te}st", + }) + }) + }) + + it("should ignore be able to save deeply nested data", async () => { + const data = { + foo: "bar", + data: [ + { cid: 1 }, + { cid: 2 }, + { + nested: { + name: "test", + ary: [1, 2, 3], + aryOfObjects: [{ a: 1 }, { b: 2 }], + }, + }, + ], + } + const query = await createQuery({ + fields: { + json: data, + extra: { + actionType: "insertOne", + }, + }, + queryVerb: "create", + }) + + const result = await config.api.query.execute(query._id!) + expect(result.data).toEqual([ + { + acknowledged: true, + insertedId: expectValidId, + }, + ]) + + await withCollection(async collection => { + const doc = await collection.findOne({ foo: { $eq: "bar" } }) + expect(doc).toEqual({ + _id: expectValidBsonObjectId, + ...data, + }) + }) + }) }) diff --git a/packages/server/src/integrations/mongodb.ts b/packages/server/src/integrations/mongodb.ts index 272810b052..c9852e4c7a 100644 --- a/packages/server/src/integrations/mongodb.ts +++ b/packages/server/src/integrations/mongodb.ts @@ -23,7 +23,7 @@ import { } from "mongodb" import environment from "../environment" -interface MongoDBConfig { +export interface MongoDBConfig { connectionString: string db: string tlsCertificateKeyFile: string @@ -348,7 +348,7 @@ const getSchema = () => { const SCHEMA: Integration = getSchema() -class MongoIntegration implements IntegrationBase { +export class MongoIntegration implements IntegrationBase { private config: MongoDBConfig private client: MongoClient diff --git a/packages/server/src/integrations/tests/mongo.spec.ts b/packages/server/src/integrations/tests/mongo.spec.ts deleted file mode 100644 index 74dd9d4ec0..0000000000 --- a/packages/server/src/integrations/tests/mongo.spec.ts +++ /dev/null @@ -1,325 +0,0 @@ -const mongo = require("mongodb") - -import { default as MongoDBIntegration } from "../mongodb" - -jest.mock("mongodb") - -class TestConfiguration { - integration: any - - constructor(config: any = {}) { - this.integration = new MongoDBIntegration.integration(config) - } -} - -describe("MongoDB Integration", () => { - let config: any - let indexName = "Users" - - beforeEach(() => { - config = new TestConfiguration() - }) - - it("calls the create method with the correct params", async () => { - const body = { - name: "Hello", - } - await config.integration.create({ - index: indexName, - json: body, - extra: { collection: "testCollection", actionType: "insertOne" }, - }) - expect(config.integration.client.insertOne).toHaveBeenCalledWith(body) - }) - - it("calls the read method with the correct params", async () => { - const query = { - json: { - address: "test", - }, - extra: { collection: "testCollection", actionType: "find" }, - } - const response = await config.integration.read(query) - expect(config.integration.client.find).toHaveBeenCalledWith(query.json) - expect(response).toEqual(expect.any(Array)) - }) - - it("calls the delete method with the correct params", async () => { - const query = { - json: { - filter: { - id: "test", - }, - options: { - opt: "option", - }, - }, - extra: { collection: "testCollection", actionType: "deleteOne" }, - } - await config.integration.delete(query) - expect(config.integration.client.deleteOne).toHaveBeenCalledWith( - query.json.filter, - query.json.options - ) - }) - - it("calls the update method with the correct params", async () => { - const query = { - json: { - filter: { - id: "test", - }, - update: { - name: "TestName", - }, - options: { - upsert: false, - }, - }, - extra: { collection: "testCollection", actionType: "updateOne" }, - } - await config.integration.update(query) - expect(config.integration.client.updateOne).toHaveBeenCalledWith( - query.json.filter, - query.json.update, - query.json.options - ) - }) - - it("throws an error when an invalid query.extra.actionType is passed for each method", async () => { - const query = { - extra: { collection: "testCollection", actionType: "deleteOne" }, - } - - let error = null - try { - await config.integration.read(query) - } catch (err) { - error = err - } - expect(error).toBeDefined() - }) - - it("creates ObjectIds if the field contains a match on ObjectId", async () => { - const query = { - json: { - filter: { - _id: "ObjectId('ACBD12345678ABCD12345678')", - name: "ObjectId('BBBB12345678ABCD12345678')", - }, - update: { - _id: "ObjectId('FFFF12345678ABCD12345678')", - name: "ObjectId('CCCC12345678ABCD12345678')", - }, - options: { - upsert: false, - }, - }, - extra: { collection: "testCollection", actionType: "updateOne" }, - } - await config.integration.update(query) - expect(config.integration.client.updateOne).toHaveBeenCalled() - - const args = config.integration.client.updateOne.mock.calls[0] - expect(args[0]).toEqual({ - _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), - name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"), - }) - expect(args[1]).toEqual({ - _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), - name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"), - }) - expect(args[2]).toEqual({ - upsert: false, - }) - }) - - it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => { - const query = { - json: { - filter: { - _id: { - $eq: "ObjectId('ACBD12345678ABCD12345678')", - }, - }, - update: { - $set: { - _id: "ObjectId('FFFF12345678ABCD12345678')", - }, - }, - options: { - upsert: true, - }, - }, - extra: { collection: "testCollection", actionType: "updateOne" }, - } - await config.integration.update(query) - expect(config.integration.client.updateOne).toHaveBeenCalled() - - const args = config.integration.client.updateOne.mock.calls[0] - expect(args[0]).toEqual({ - _id: { - $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), - }, - }) - expect(args[1]).toEqual({ - $set: { - _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"), - }, - }) - expect(args[2]).toEqual({ - upsert: true, - }) - }) - - it("supports findOneAndUpdate", async () => { - const query = { - json: { - filter: { - _id: { - $eq: "ObjectId('ACBD12345678ABCD12345678')", - }, - }, - update: { - $set: { - name: "UPDATED", - age: 99, - }, - }, - options: { - upsert: false, - }, - }, - extra: { collection: "testCollection", actionType: "findOneAndUpdate" }, - } - await config.integration.read(query) - expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled() - - const args = config.integration.client.findOneAndUpdate.mock.calls[0] - expect(args[0]).toEqual({ - _id: { - $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), - }, - }) - expect(args[1]).toEqual({ - $set: { - name: "UPDATED", - age: 99, - }, - }) - expect(args[2]).toEqual({ - upsert: false, - includeResultMetadata: true, - }) - }) - - it("can parse nested objects with arrays", async () => { - const query = { - json: `{ - "_id": { - "$eq": "ObjectId('ACBD12345678ABCD12345678')" - } - }, - { - "$set": { - "value": { - "data": [ - { "cid": 1 }, - { "cid": 2 }, - { "nested": { - "name": "test" - }} - ] - } - } - }, - { - "upsert": true - }`, - extra: { collection: "testCollection", actionType: "updateOne" }, - } - await config.integration.update(query) - expect(config.integration.client.updateOne).toHaveBeenCalled() - - const args = config.integration.client.updateOne.mock.calls[0] - expect(args[0]).toEqual({ - _id: { - $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), - }, - }) - expect(args[1]).toEqual({ - $set: { - value: { - data: [ - { cid: 1 }, - { cid: 2 }, - { - nested: { - name: "test", - }, - }, - ], - }, - }, - }) - expect(args[2]).toEqual({ - upsert: true, - }) - }) - - it("ignores braces within strings when parsing nested objects", async () => { - const query = { - json: `{ - "_id": { - "$eq": "ObjectId('ACBD12345678ABCD12345678')" - } - }, - { - "$set": { - "value": { - "data": [ - { "cid": 1 }, - { "cid": 2 }, - { "nested": { - "name": "te}st" - }} - ] - } - } - }, - { - "upsert": true, - "extra": "ad\\"{\\"d" - }`, - extra: { collection: "testCollection", actionType: "updateOne" }, - } - await config.integration.update(query) - expect(config.integration.client.updateOne).toHaveBeenCalled() - - const args = config.integration.client.updateOne.mock.calls[0] - expect(args[0]).toEqual({ - _id: { - $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"), - }, - }) - expect(args[1]).toEqual({ - $set: { - value: { - data: [ - { cid: 1 }, - { cid: 2 }, - { - nested: { - name: "te}st", - }, - }, - ], - }, - }, - }) - expect(args[2]).toEqual({ - upsert: true, - extra: 'ad"{"d', - }) - }) -}) diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index 32866314ff..7b887c3fb6 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -5,7 +5,7 @@ import { PreviewQueryRequest, PreviewQueryResponse, } from "@budibase/types" -import { TestAPI } from "./base" +import { Expectations, TestAPI } from "./base" export class QueryAPI extends TestAPI { create = async (body: Query): Promise => { @@ -14,12 +14,14 @@ export class QueryAPI extends TestAPI { execute = async ( queryId: string, - body?: ExecuteQueryRequest + body?: ExecuteQueryRequest, + expectations?: Expectations ): Promise => { return await this._post( `/api/v2/queries/${queryId}`, { body, + expectations, } ) } From 80cc83d2f07947ed78a9ecb0b71c3edd0f9f7225 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 15:42:28 +0000 Subject: [PATCH 04/28] Fix huge amount of wasted renders by observing entire component context when generating form block fields --- .../client/src/components/app/blocks/form/FormBlock.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 8c084a71ab..d249569731 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -34,6 +34,7 @@ $: formattedFields = convertOldFieldFormat(fields) $: fieldsOrDefault = getDefaultFields(formattedFields, schema) $: fetchSchema(dataSource) + $: id = $component.id // We could simply spread $$props into the inner form and append our // additions, but that would create svelte warnings about unused props and // make maintenance in future more confusing as we typically always have a @@ -53,7 +54,7 @@ buttons: buttons || Utils.buildFormBlockButtonConfig({ - _id: $component.id, + _id: id, showDeleteButton, showSaveButton, saveButtonLabel, From db4c3eb214070ef3177e6e4a9b98bf76972e688b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 20 Mar 2024 15:42:40 +0000 Subject: [PATCH 05/28] Remove no longer needed jest.unmnock call. --- packages/server/src/api/routes/tests/queries/mongodb.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index 14ee46c535..e0351e0ce3 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -3,8 +3,6 @@ import * as setup from "../utilities" import { databaseTestProviders } from "../../../../integrations/tests/utils" import { MongoClient, type Collection, BSON } from "mongodb" -jest.unmock("mongodb") - const collection = "test_collection" const expectValidId = expect.stringMatching(/^\w{24}$/) From fdde7f889581244ba1a9460364223b3da6fd5eb0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 15:46:07 +0000 Subject: [PATCH 06/28] Improve performance of multistep form block --- .../app/blocks/MultiStepFormblock.svelte | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte index 762afc1579..f47655882d 100644 --- a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte +++ b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte @@ -26,9 +26,11 @@ let schema + $: id = $component.id + $: selected = $component.selected $: fetchSchema(dataSource) - $: enrichedSteps = enrichSteps(steps, schema, $component.id, $currentStep) - $: updateCurrentStep(enrichedSteps, $builderStore, $component) + $: updateCurrentStep(enrichedSteps, $builderStore, id, selected) + $: enrichedSteps = enrichSteps(steps, schema, id, $currentStep) // Provide additional data context for live binding eval export const getAdditionalDataContext = () => { @@ -40,16 +42,12 @@ } } - const updateCurrentStep = (steps, builderStore, component) => { + const updateCurrentStep = (steps, builderStore, id, selected) => { const { componentId, step } = builderStore.metadata || {} // If we aren't in the builder or aren't selected then don't update the step // context at all, allowing the normal form to take control. - if ( - !component.selected || - !builderStore.inBuilder || - componentId !== component.id - ) { + if (!selected || !builderStore.inBuilder || componentId !== id) { return } From 65b4b6c35d58db9a44f13cedb1d842bb281fcc8e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 15:46:15 +0000 Subject: [PATCH 07/28] Improve performance of table block --- .../components/app/blocks/TableBlock.svelte | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index 7c58f90508..c371bed147 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -48,20 +48,7 @@ let schemaLoaded = false $: deleteLabel = setDeleteLabel(sidePanelDeleteLabel, sidePanelShowDelete) - - const setDeleteLabel = sidePanelDeleteLabel => { - // Accommodate old config to ensure delete button does not reappear - let labelText = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel - - // Empty text is considered hidden. - if (labelText?.trim() === "") { - return "" - } - - // Default to "Delete" if the value is unset - return labelText || "Delete" - } - + $: id = $component.id $: isDSPlus = dataSource?.type === "table" || dataSource?.type === "viewV2" $: fetchSchema(dataSource) $: enrichSearchColumns(searchColumns, schema).then( @@ -105,6 +92,19 @@ }, ] + const setDeleteLabel = sidePanelDeleteLabel => { + // Accommodate old config to ensure delete button does not reappear + let labelText = sidePanelShowDelete === false ? "" : sidePanelDeleteLabel + + // Empty text is considered hidden. + if (labelText?.trim() === "") { + return "" + } + + // Default to "Delete" if the value is unset + return labelText || "Delete" + } + // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource?.type === "table") { @@ -267,7 +267,7 @@ dataSource, buttonPosition: "top", buttons: Utils.buildFormBlockButtonConfig({ - _id: $component.id + "-form-edit", + _id: id + "-form-edit", showDeleteButton: deleteLabel !== "", showSaveButton: true, saveButtonLabel: sidePanelSaveLabel || "Save", @@ -301,7 +301,7 @@ dataSource, buttonPosition: "top", buttons: Utils.buildFormBlockButtonConfig({ - _id: $component.id + "-form-new", + _id: id + "-form-new", showDeleteButton: false, showSaveButton: true, saveButtonLabel: "Save", From 9149828c7c12b6e2651827c2595b8f0964adfe5c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 15:50:23 +0000 Subject: [PATCH 08/28] Improve performance of multistep form block even more and fix missing params --- .../app/blocks/MultiStepFormblock.svelte | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte index f47655882d..5cb3640355 100644 --- a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte +++ b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte @@ -28,9 +28,11 @@ $: id = $component.id $: selected = $component.selected + $: inBuilder = $builderStore.inBuilder + $: builderStep = $builderStore.metadata?.step $: fetchSchema(dataSource) - $: updateCurrentStep(enrichedSteps, $builderStore, id, selected) - $: enrichedSteps = enrichSteps(steps, schema, id, $currentStep) + $: enrichedSteps = enrichSteps(steps, schema, id) + $: updateCurrentStep(enrichedSteps, selected, inBuilder, builderStep) // Provide additional data context for live binding eval export const getAdditionalDataContext = () => { @@ -42,17 +44,15 @@ } } - const updateCurrentStep = (steps, builderStore, id, selected) => { - const { componentId, step } = builderStore.metadata || {} - + const updateCurrentStep = (steps, selected, inBuilder, builderStep) => { // If we aren't in the builder or aren't selected then don't update the step // context at all, allowing the normal form to take control. - if (!selected || !builderStore.inBuilder || componentId !== id) { + if (!selected || !inBuilder) { return } // Ensure we have a valid step selected - let newStep = Math.min(step || 0, steps.length - 1) + let newStep = Math.min(builderStep || 0, steps.length - 1) // Sanity check newStep = Math.max(newStep, 0) @@ -61,7 +61,7 @@ currentStep.set(newStep + 1) } - const fetchSchema = async () => { + const fetchSchema = async dataSource => { schema = (await fetchDatasourceSchema(dataSource)) || {} } From c58ac5810ecd3ca354ade57add19f78f146ce5e9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 15:53:53 +0000 Subject: [PATCH 09/28] Simplify reactive logic --- .../components/app/blocks/MultiStepFormblock.svelte | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte index 5cb3640355..bcc62b5229 100644 --- a/packages/client/src/components/app/blocks/MultiStepFormblock.svelte +++ b/packages/client/src/components/app/blocks/MultiStepFormblock.svelte @@ -28,11 +28,10 @@ $: id = $component.id $: selected = $component.selected - $: inBuilder = $builderStore.inBuilder $: builderStep = $builderStore.metadata?.step $: fetchSchema(dataSource) $: enrichedSteps = enrichSteps(steps, schema, id) - $: updateCurrentStep(enrichedSteps, selected, inBuilder, builderStep) + $: updateCurrentStep(enrichedSteps, selected, builderStep) // Provide additional data context for live binding eval export const getAdditionalDataContext = () => { @@ -44,17 +43,15 @@ } } - const updateCurrentStep = (steps, selected, inBuilder, builderStep) => { - // If we aren't in the builder or aren't selected then don't update the step - // context at all, allowing the normal form to take control. - if (!selected || !inBuilder) { + const updateCurrentStep = (steps, selected, builderStep) => { + // If we aren't selected in the builder then just allowing the normal form + // to take control. + if (!selected) { return } // Ensure we have a valid step selected let newStep = Math.min(builderStep || 0, steps.length - 1) - - // Sanity check newStep = Math.max(newStep, 0) // Add 1 because the form component expects 1 indexed rather than 0 indexed From fe3fea3b1d8fa830858e11d0cbd71107414198b1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 20 Mar 2024 16:01:12 +0000 Subject: [PATCH 10/28] Update pro and account portal references. --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 6465dc9c2a..f5b467b6b1 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 6465dc9c2a38e1380b32204cad4ae0c1f33e065a +Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a diff --git a/packages/pro b/packages/pro index 8baf8586ec..dd748e045f 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 8baf8586ec078951230c8466d5f13f9b6d5ed055 +Subproject commit dd748e045ffdbc6662c5d2b76075f01d65a96a2f From 5acfc3143da29b84c3d38bcc0f49657c9ed68653 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 20 Mar 2024 17:01:23 +0000 Subject: [PATCH 11/28] Multiple performance improvements to component selection and hovering --- packages/builder/src/stores/builder/hover.js | 13 +++++++++++++ .../components/preview/HoverIndicator.svelte | 5 ++--- .../src/components/preview/Indicator.svelte | 6 +----- .../src/components/preview/IndicatorSet.svelte | 1 + packages/client/src/index.js | 2 +- packages/client/src/stores/hover.js | 18 ++++++++++++++++-- 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/stores/builder/hover.js b/packages/builder/src/stores/builder/hover.js index f0444a7da5..98cdc9e416 100644 --- a/packages/builder/src/stores/builder/hover.js +++ b/packages/builder/src/stores/builder/hover.js @@ -7,12 +7,25 @@ export const INITIAL_HOVER_STATE = { } export class HoverStore extends BudiStore { + hoverTimeout + constructor() { super({ ...INITIAL_HOVER_STATE }) this.hover = this.hover.bind(this) } hover(componentId, notifyClient = true) { + clearTimeout(this.hoverTimeout) + if (componentId) { + this.processHover(componentId, notifyClient) + } else { + this.hoverTimeout = setTimeout(() => { + this.processHover(componentId, notifyClient) + }, 10) + } + } + + processHover(componentId, notifyClient) { if (componentId === get(this.store).componentId) { return } diff --git a/packages/client/src/components/preview/HoverIndicator.svelte b/packages/client/src/components/preview/HoverIndicator.svelte index 164ec32ab4..69e13e4219 100644 --- a/packages/client/src/components/preview/HoverIndicator.svelte +++ b/packages/client/src/components/preview/HoverIndicator.svelte @@ -1,10 +1,9 @@
x.visible) $: offset = $builderStore.inBuilder ? 0 : 2 + $: componentId, debouncedUpdate() let updating = false let observers = [] diff --git a/packages/client/src/index.js b/packages/client/src/index.js index 9c249dd5b3..de6d1bdc12 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -98,7 +98,7 @@ const loadBudibase = async () => { context: stringifiedContext, }) } else if (type === "hover-component") { - hoverStore.actions.hoverComponent(data) + hoverStore.actions.hoverComponent(data, false) } else if (type === "builder-meta") { builderStore.actions.setMetadata(data) } diff --git a/packages/client/src/stores/hover.js b/packages/client/src/stores/hover.js index 24f315a126..014a9f1aa0 100644 --- a/packages/client/src/stores/hover.js +++ b/packages/client/src/stores/hover.js @@ -5,13 +5,27 @@ const createHoverStore = () => { const store = writable({ hoveredComponentId: null, }) + let hoverTimeout - const hoverComponent = id => { + const hoverComponent = (id, notifyBuilder = true) => { + clearTimeout(hoverTimeout) + if (id) { + processHover(id, notifyBuilder) + } else { + hoverTimeout = setTimeout(() => { + processHover(id, notifyBuilder) + }, 10) + } + } + + const processHover = (id, notifyBuilder = true) => { if (id === get(store).hoveredComponentId) { return } store.set({ hoveredComponentId: id }) - eventStore.actions.dispatchEvent("hover-component", { id }) + if (notifyBuilder) { + eventStore.actions.dispatchEvent("hover-component", { id }) + } } return { From e7dd137b28a88d6d01757ccf765024a70a9a628b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 09:14:01 +0000 Subject: [PATCH 12/28] Add memo and derivedMemo into SDK --- packages/client/src/sdk.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js index d86d635970..1f996bf656 100644 --- a/packages/client/src/sdk.js +++ b/packages/client/src/sdk.js @@ -34,6 +34,8 @@ import { LuceneUtils, Constants, RowUtils, + memo, + derivedMemo, } from "@budibase/frontend-core" export default { @@ -71,6 +73,8 @@ export default { makePropSafe, createContextStore, generateGoldenSample: RowUtils.generateGoldenSample, + memo, + derivedMemo, // Components Provider, From 96f7fe6b125ab6bbf325ebe3bf53dfda0b6dc52d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 09:27:42 +0000 Subject: [PATCH 13/28] Remove grid layout component --- .../[componentId]/new/_components/componentStructure.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json index 96e8faf93c..4edd56dbd2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json @@ -16,7 +16,7 @@ { "name": "Layout", "icon": "ClassicGridView", - "children": ["container", "section", "grid", "sidepanel"] + "children": ["container", "section", "sidepanel"] }, { "name": "Data", From 7d3153601423d83aa3d566a35364852caa1b85cd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 12:08:24 +0000 Subject: [PATCH 14/28] Fix issue with click_outside where non-function types could be stored as callbacks --- packages/bbui/src/Actions/click_outside.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js index 2c54427b3a..eafca657f3 100644 --- a/packages/bbui/src/Actions/click_outside.js +++ b/packages/bbui/src/Actions/click_outside.js @@ -79,7 +79,8 @@ const removeHandler = id => { export default (element, opts) => { const id = Math.random() const update = newOpts => { - const callback = newOpts?.callback || newOpts + const callback = + newOpts?.callback || (typeof newOpts === "function" ? newOpts : null) const anchor = newOpts?.anchor || element const allowedType = newOpts?.allowedType || "click" updateHandler(id, element, anchor, callback, allowedType) From 006a58dae8ceb67bc0b1807eac279175deed01ed Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 21 Mar 2024 13:00:39 +0000 Subject: [PATCH 15/28] Update pro and account portal references. --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 6465dc9c2a..f5b467b6b1 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 6465dc9c2a38e1380b32204cad4ae0c1f33e065a +Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a diff --git a/packages/pro b/packages/pro index 8baf8586ec..dd748e045f 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 8baf8586ec078951230c8466d5f13f9b6d5ed055 +Subproject commit dd748e045ffdbc6662c5d2b76075f01d65a96a2f From 07f8e1981a3527e179be0149c07f6c50f63946c1 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 13:22:03 +0000 Subject: [PATCH 16/28] Improve client indicators by properly caching all properties to avoid inconsistent and stale states --- .../src/components/preview/DNDHandler.svelte | 3 +- .../components/preview/HoverIndicator.svelte | 7 +- .../src/components/preview/Indicator.svelte | 6 -- .../components/preview/IndicatorSet.svelte | 99 ++++++++++--------- .../preview/SelectionIndicator.svelte | 3 +- 5 files changed, 60 insertions(+), 58 deletions(-) diff --git a/packages/client/src/components/preview/DNDHandler.svelte b/packages/client/src/components/preview/DNDHandler.svelte index e93c42b863..767efc9e3a 100644 --- a/packages/client/src/components/preview/DNDHandler.svelte +++ b/packages/client/src/components/preview/DNDHandler.svelte @@ -345,8 +345,7 @@ diff --git a/packages/client/src/components/preview/HoverIndicator.svelte b/packages/client/src/components/preview/HoverIndicator.svelte index 69e13e4219..d204d77f49 100644 --- a/packages/client/src/components/preview/HoverIndicator.svelte +++ b/packages/client/src/components/preview/HoverIndicator.svelte @@ -1,9 +1,11 @@
({ + // Cached props + componentId, + color, + zIndex, + prefix, + allowResizeAnchors, + + // Computed state + indicators: [], + text: null, + icon: null, + insideGrid: false, + error: false, + }) + let interval - let text - let icon - let insideGrid = false - let errorState = false - - $: visibleIndicators = indicators.filter(x => x.visible) - $: offset = $builderStore.inBuilder ? 0 : 2 - $: componentId, debouncedUpdate() - + let state = defaultState() + let nextState = null let updating = false let observers = [] let callbackCount = 0 - let nextIndicators = [] + + $: visibleIndicators = state.indicators.filter(x => x.visible) + $: offset = $builderStore.inBuilder ? 0 : 2 + $: $$props, debouncedUpdate() const checkInsideGrid = id => { const component = document.getElementsByClassName(id)[0] @@ -45,10 +56,10 @@ if (callbackCount >= observers.length) { return } - nextIndicators[idx].visible = - nextIndicators[idx].insideSidePanel || entries[0].isIntersecting + nextState.indicators[idx].visible = + nextState.indicators[idx].insideSidePanel || entries[0].isIntersecting if (++callbackCount === observers.length) { - indicators = nextIndicators + state = nextState updating = false } } @@ -60,7 +71,7 @@ // Sanity check if (!componentId) { - indicators = [] + state = defaultState() return } @@ -69,25 +80,25 @@ callbackCount = 0 observers.forEach(o => o.disconnect()) observers = [] - nextIndicators = [] + nextState = defaultState() // Check if we're inside a grid if (allowResizeAnchors) { - insideGrid = checkInsideGrid(componentId) + nextState.insideGrid = checkInsideGrid(componentId) } // Determine next set of indicators const parents = document.getElementsByClassName(componentId) if (parents.length) { - text = parents[0].dataset.name - if (prefix) { - text = `${prefix} ${text}` + nextState.text = parents[0].dataset.name + if (nextState.prefix) { + nextState.text = `${nextState.prefix} ${nextState.text}` } if (parents[0].dataset.icon) { - icon = parents[0].dataset.icon + nextState.icon = parents[0].dataset.icon } } - errorState = parents?.[0]?.classList.contains("error") + nextState.error = parents?.[0]?.classList.contains("error") // Batch reads to minimize reflow const scrollX = window.scrollX @@ -103,8 +114,9 @@ // If there aren't any nodes then reset if (!children.length) { - indicators = [] + state = defaultState() updating = false + return } const device = document.getElementById("app-root") @@ -120,7 +132,7 @@ observers.push(observer) const elBounds = child.getBoundingClientRect() - nextIndicators.push({ + nextState.indicators.push({ top: elBounds.top + scrollY - deviceBounds.top - offset, left: elBounds.left + scrollX - deviceBounds.left - offset, width: elBounds.width + 4, @@ -145,20 +157,17 @@ }) -{#key componentId} - {#each visibleIndicators as indicator, idx} - - {/each} -{/key} +{#each visibleIndicators as indicator, idx} + +{/each} diff --git a/packages/client/src/components/preview/SelectionIndicator.svelte b/packages/client/src/components/preview/SelectionIndicator.svelte index bca0341628..a271389cbd 100644 --- a/packages/client/src/components/preview/SelectionIndicator.svelte +++ b/packages/client/src/components/preview/SelectionIndicator.svelte @@ -10,7 +10,6 @@ From 8a9bb0294b6f2208fce280569e4d1ee975af0282 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 15:22:07 +0000 Subject: [PATCH 17/28] Allow horizontal overflow in drawer contents --- packages/bbui/src/Drawer/DrawerContent.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/bbui/src/Drawer/DrawerContent.svelte b/packages/bbui/src/Drawer/DrawerContent.svelte index 490dfecc31..f7345afb11 100644 --- a/packages/bbui/src/Drawer/DrawerContent.svelte +++ b/packages/bbui/src/Drawer/DrawerContent.svelte @@ -42,7 +42,6 @@ .main { height: 100%; overflow: auto; - overflow-x: hidden; } .padding .main { padding: var(--spacing-xl); From c6fca37b43c59a6be56d1383d6a2facc08d437ba Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 16:05:02 +0000 Subject: [PATCH 18/28] Update dataprovider validation which was not aware of global bindings --- packages/builder/src/stores/builder/components.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 0d7da1ba58..a8522db80c 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -11,6 +11,7 @@ import { findComponentParent, findAllMatchingComponents, makeComponentUnique, + findComponentType, } from "helpers/components" import { getComponentFieldOptions } from "helpers/formFields" import { selectedScreen } from "./screens" @@ -279,12 +280,10 @@ export class ComponentStore extends BudiStore { else { if (setting.type === "dataProvider") { // Validate data provider exists, or else clear it - const treeId = parent?._id || component._id - const path = findComponentPath(screen?.props, treeId) - const providers = path.filter(component => - component._component?.endsWith("/dataprovider") + const providers = findAllMatchingComponents( + screen?.props, + x => x._component === "@budibase/standard-components/dataprovider" ) - // Validate non-empty values const valid = providers?.some(dp => value.includes?.(dp._id)) if (!valid) { if (providers.length) { From d3535a255d6d441c6877088fcc6d640ce4904f54 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 16:24:35 +0000 Subject: [PATCH 19/28] Respect ordering of options when using a data provider options source --- .../src/components/app/forms/optionsParser.js | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/client/src/components/app/forms/optionsParser.js b/packages/client/src/components/app/forms/optionsParser.js index bd69967731..3bb5865774 100644 --- a/packages/client/src/components/app/forms/optionsParser.js +++ b/packages/client/src/components/app/forms/optionsParser.js @@ -6,36 +6,24 @@ export const getOptions = ( valueColumn, customOptions ) => { - const isArray = fieldSchema?.type === "array" // Take options from schema if (optionsSource == null || optionsSource === "schema") { return fieldSchema?.constraints?.inclusion ?? [] } - if (optionsSource === "provider" && isArray) { - let optionsSet = {} - - dataProvider?.rows?.forEach(row => { - const value = row?.[valueColumn] - if (value != null) { - const label = row[labelColumn] || value - optionsSet[value] = { value, label } - } - }) - return Object.values(optionsSet) - } - // Extract options from data provider if (optionsSource === "provider" && valueColumn) { - let optionsSet = {} + let valueCache = {} + let options = [] dataProvider?.rows?.forEach(row => { const value = row?.[valueColumn] - if (value != null) { + if (value != null && !valueCache[value]) { + valueCache[value] = true const label = row[labelColumn] || value - optionsSet[value] = { value, label } + options.push({ value, label }) } }) - return Object.values(optionsSet) + return options } // Extract custom options From 44114d9af10876c85334d6ea221077862bbb51d7 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 21 Mar 2024 16:26:53 +0000 Subject: [PATCH 20/28] Lint --- packages/builder/src/stores/builder/components.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index a8522db80c..fe5f4e8a05 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -11,7 +11,6 @@ import { findComponentParent, findAllMatchingComponents, makeComponentUnique, - findComponentType, } from "helpers/components" import { getComponentFieldOptions } from "helpers/formFields" import { selectedScreen } from "./screens" From 732c715498470e8557952392d483bf4755b8a6b9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 22 Mar 2024 12:09:55 +0000 Subject: [PATCH 21/28] Add snippets into context when transforming column values legacy tables --- packages/bbui/src/Table/CellRenderer.svelte | 5 ++++- packages/bbui/src/Table/Table.svelte | 2 ++ .../client/src/components/app/deprecated/table/Table.svelte | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/bbui/src/Table/CellRenderer.svelte b/packages/bbui/src/Table/CellRenderer.svelte index 4ad6e22d7e..6f6e210fe1 100644 --- a/packages/bbui/src/Table/CellRenderer.svelte +++ b/packages/bbui/src/Table/CellRenderer.svelte @@ -12,6 +12,7 @@ export let schema export let value export let customRenderers = [] + export let snippets let renderer const typeMap = { @@ -32,6 +33,8 @@ $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer $: cellValue = getCellValue(value, schema.template) + $: console.log(snippets) + const getType = schema => { // Use a string renderer for dates if we use a custom template if (schema?.type === "datetime" && schema?.template) { @@ -44,7 +47,7 @@ if (!template) { return value } - return processStringSync(template, { value }) + return processStringSync(template, { value, snippets }) } diff --git a/packages/bbui/src/Table/Table.svelte b/packages/bbui/src/Table/Table.svelte index 33b9bd9a7e..868f7b3a0b 100644 --- a/packages/bbui/src/Table/Table.svelte +++ b/packages/bbui/src/Table/Table.svelte @@ -42,6 +42,7 @@ export let customPlaceholder = false export let showHeaderBorder = true export let placeholderText = "No rows found" + export let snippets = [] const dispatch = createEventDispatcher() @@ -425,6 +426,7 @@ Date: Fri, 22 Mar 2024 12:10:22 +0000 Subject: [PATCH 22/28] Fix code mirror autocomplete showing NO NAME for certain bindings --- packages/builder/src/components/common/CodeEditor/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/CodeEditor/index.js b/packages/builder/src/components/common/CodeEditor/index.js index f66c84adce..a04a140ad6 100644 --- a/packages/builder/src/components/common/CodeEditor/index.js +++ b/packages/builder/src/components/common/CodeEditor/index.js @@ -313,7 +313,7 @@ export const bindingsToCompletions = (bindings, mode) => { ...bindingByCategory[catKey].reduce((acc, binding) => { let displayType = binding.fieldSchema?.type || binding.display?.type acc.push({ - label: binding.display?.name || "NO NAME", + label: binding.display?.name || binding.readableBinding || "NO NAME", info: completion => { return buildBindingInfoNode(completion, binding) }, From 53cd48e8663a12d6d87dfaa26cb6665ec4772ebe Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 22 Mar 2024 12:11:03 +0000 Subject: [PATCH 23/28] Add value for 'value' binding when editing table columns to improve live eval --- .../components/common/bindings/ClientBindingPanel.svelte | 3 ++- .../settings/controls/ColumnEditor/CellDrawer.svelte | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte index cb65d2bbe4..4e5789b563 100644 --- a/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/ClientBindingPanel.svelte @@ -8,6 +8,7 @@ export let allowJS = false export let allowHelpers = true export let autofocusEditor = false + export let context = null $: enrichedBindings = enrichBindings(bindings) @@ -27,7 +28,7 @@ @@ -41,6 +44,9 @@ icon: "TableColumnMerge", }, ]} + context={{ + value: columnValue, + }} /> From a64738fa253a8bcaf7df57267d1e00026740d173 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 22 Mar 2024 12:14:59 +0000 Subject: [PATCH 24/28] Provide live eval context from legacy table components --- .../app/deprecated/table/Table.svelte | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/client/src/components/app/deprecated/table/Table.svelte b/packages/client/src/components/app/deprecated/table/Table.svelte index 71f3cc1712..ac3d88d29c 100644 --- a/packages/client/src/components/app/deprecated/table/Table.svelte +++ b/packages/client/src/components/app/deprecated/table/Table.svelte @@ -17,8 +17,14 @@ const component = getContext("component") const context = getContext("context") - const { styleable, getAction, ActionTypes, rowSelectionStore } = - getContext("sdk") + const { + styleable, + getAction, + ActionTypes, + rowSelectionStore, + generateGoldenSample, + } = getContext("sdk") + const customColumnKey = `custom-${Math.random()}` const customRenderers = [ { @@ -63,6 +69,16 @@ selectedRows, } + // Provide additional data context for live binding eval + export const getAdditionalDataContext = () => { + const goldenRow = generateGoldenSample(data) + return { + eventContext: { + row: goldenRow, + }, + } + } + const getFields = ( schema, customColumns, From 9072142ca948a0922915e53b0429719ead04b5fd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 22 Mar 2024 12:31:35 +0000 Subject: [PATCH 25/28] Remove log --- packages/bbui/src/Table/CellRenderer.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/bbui/src/Table/CellRenderer.svelte b/packages/bbui/src/Table/CellRenderer.svelte index 6f6e210fe1..eff1178f6d 100644 --- a/packages/bbui/src/Table/CellRenderer.svelte +++ b/packages/bbui/src/Table/CellRenderer.svelte @@ -33,8 +33,6 @@ $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer $: cellValue = getCellValue(value, schema.template) - $: console.log(snippets) - const getType = schema => { // Use a string renderer for dates if we use a custom template if (schema?.type === "datetime" && schema?.template) { From db19f2040f60b1aa756e71570a9d9cb62682a5cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 22 Mar 2024 15:52:46 +0000 Subject: [PATCH 26/28] Disabling the single attachment column toggle for now - until we add the new type. --- .../DataTable/modals/CreateEditColumn.svelte | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index d39a6a92a7..c440a4172e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -710,21 +710,6 @@ thin text="Allow multiple users" /> - {:else if editableColumn.type === FieldType.ATTACHMENT} - { - if (!e.detail) { - editableColumn.subtype = FieldTypeSubtypes.ATTACHMENT.SINGLE - } else { - delete editableColumn.subtype - } - }} - thin - text="Allow multiple" - /> {/if} {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}