From 6bc88d8c797e758f7e499301b8ea26075d148068 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 8 Oct 2024 11:39:56 +0200 Subject: [PATCH 001/192] yarn add zod --- packages/types/package.json | 3 +- yarn.lock | 281 ++++++++++++++++++++---------------- 2 files changed, 162 insertions(+), 122 deletions(-) diff --git a/packages/types/package.json b/packages/types/package.json index bce25c7086..7e0f98b367 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -22,6 +22,7 @@ "typescript": "5.5.2" }, "dependencies": { - "scim-patch": "^0.8.1" + "scim-patch": "^0.8.1", + "zod": "^3.23.8" } } diff --git a/yarn.lock b/yarn.lock index 1198e98ad6..bc18155a9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -764,20 +764,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@4.2.1", "@azure/identity@^3.4.1": - version "4.2.1" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" - integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== +"@azure/identity@^3.4.1": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" + integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.3.0" + "@azure/core-util" "^1.6.1" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.11.1" - "@azure/msal-node" "^2.9.2" + "@azure/msal-browser" "^3.5.0" + "@azure/msal-node" "^2.5.1" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -808,10 +808,10 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.11.1": - version "3.24.0" - resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.24.0.tgz#3208047672d0b0c943b0bef5f995d510d6582ae4" - integrity sha512-JGNV9hTYAa7lsum9IMIibn2kKczAojNihGo1hi7pG0kNrcKej530Fl6jxwM05A44/6I079CSn6WxYxbVhKUmWg== +"@azure/msal-browser@^3.5.0": + version "3.25.0" + resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.25.0.tgz#7ce0949977bc9e0c58319f7090c44fe5537104d4" + integrity sha512-a0Y7pmSy8SC1s9bvwr+REvyAA1nQcITlZvkElM2gNUPYFTTNUTEdcpg73TmawNucyMdZ9xb/GFcuhrLOqYAzwg== dependencies: "@azure/msal-common" "14.15.0" @@ -820,10 +820,10 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.15.0.tgz#0e27ac0bb88fe100f4f8d1605b64d5c268636a55" integrity sha512-ImAQHxmpMneJ/4S8BRFhjt1MZ3bppmpRPYYNyzeQPeFN288YKbb8TmmISQEbtfkQ1BPASvYZU5doIZOPBAqENQ== -"@azure/msal-node@^2.9.2": - version "2.14.0" - resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.14.0.tgz#7881895d41b03d8b9b38a29550ba3bbb15f73b3c" - integrity sha512-rrfzIpG3Q1rHjVYZmHAEDidWAZZ2cgkxlIcMQ8dHebRISaZ2KCV33Q8Vs+uaV6lxweROabNxKFlR2lIKagZqYg== +"@azure/msal-node@^2.5.1": + version "2.15.0" + resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.15.0.tgz#50bf8e692a6656027c073a75d877a8a478aafdfd" + integrity sha512-gVPW8YLz92ZeCibQH2QUw96odJoiM3k/ZPH3f2HxptozmH6+OnyyvKXo/Egg39HAM230akarQKHf0W74UHlh0Q== dependencies: "@azure/msal-common" "14.15.0" jsonwebtoken "^9.0.0" @@ -2051,44 +2051,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.32.11": - version "0.0.0" - dependencies: - "@budibase/nano" "10.1.5" - "@budibase/pouchdb-replication-stream" "1.2.11" - "@budibase/shared-core" "0.0.0" - "@budibase/types" "0.0.0" - aws-cloudfront-sign "3.0.2" - aws-sdk "2.1030.0" - bcrypt "5.1.0" - bcryptjs "2.4.3" - bull "4.10.1" - correlation-id "4.0.0" - dd-trace "5.2.0" - dotenv "16.0.1" - ioredis "5.3.2" - joi "17.6.0" - jsonwebtoken "9.0.2" - knex "2.4.2" - koa-passport "^6.0.0" - koa-pino-logger "4.0.0" - lodash "4.17.21" - node-fetch "2.6.7" - passport-google-oauth "2.0.0" - passport-local "1.0.0" - passport-oauth2-refresh "^2.1.0" - pino "8.11.0" - pino-http "8.3.3" - posthog-node "4.0.1" - pouchdb "7.3.0" - pouchdb-find "7.2.2" - redlock "4.2.0" - rotating-file-stream "3.1.0" - sanitize-s3-objectkey "0.0.1" - semver "^7.5.4" - tar-fs "2.1.1" - uuid "^8.3.2" - "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2131,47 +2093,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "2.32.11" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d" - integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA== - dependencies: - "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "2.32.11" - "@budibase/shared-core" "2.32.11" - "@budibase/string-templates" "2.32.11" - "@budibase/types" "2.32.11" - "@koa/router" "8.0.8" - bull "4.10.1" - dd-trace "5.2.0" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - openai "4.59.0" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - -"@budibase/shared-core@2.32.11": - version "0.0.0" - dependencies: - "@budibase/types" "0.0.0" - cron-validate "1.4.5" - -"@budibase/string-templates@2.32.11": - version "0.0.0" - dependencies: - "@budibase/handlebars-helpers" "^0.13.2" - dayjs "^1.10.8" - handlebars "^4.7.8" - lodash.clonedeep "^4.5.0" - -"@budibase/types@2.32.11": - version "0.0.0" - dependencies: - scim-patch "^0.8.1" - "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7326,7 +7247,23 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11126,7 +11063,14 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5: + version "4.2.5" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" + integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== + dependencies: + strnum "^1.0.5" + +fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11375,6 +11319,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12457,7 +12406,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -12977,6 +12931,11 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-builtin-module@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" @@ -13453,11 +13412,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14362,7 +14316,14 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@^3.0.2, kind-of@^3.1.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -15982,7 +15943,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16176,13 +16137,25 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17318,7 +17291,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18578,7 +18559,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19613,6 +19594,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19685,13 +19671,33 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21270,7 +21276,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21280,6 +21286,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21795,14 +21809,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22586,10 +22592,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22599,6 +22628,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" @@ -22846,3 +22880,8 @@ zip-stream@^6.0.1: archiver-utils "^5.0.0" compress-commons "^6.0.2" readable-stream "^4.0.0" + +zod@^3.23.8: + version "3.23.8" + resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" + integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== From 5a8aba7f523f8c6a2a877121128257285992bbad Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 8 Oct 2024 11:41:04 +0200 Subject: [PATCH 002/192] zod validator --- .../server/src/middleware/zod-validator.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 packages/server/src/middleware/zod-validator.ts diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts new file mode 100644 index 0000000000..6fb338e5e8 --- /dev/null +++ b/packages/server/src/middleware/zod-validator.ts @@ -0,0 +1,33 @@ +import { AnyZodObject } from "zod" +import { Ctx } from "@budibase/types" + +function validate(schema: AnyZodObject, property: "body" | "params") { + // Return a Koa middleware function + return (ctx: Ctx, next: any) => { + if (!schema) { + return next() + } + let params = null + if (ctx[property] != null) { + params = ctx[property] + } else if (property === "body" && ctx.request[property] != null) { + params = ctx.request[property] + } else if (ctx.request.get(property) != null) { + params = ctx.request.get(property) + } + + const { error } = schema.safeParse(params) + if (error) { + ctx.throw(400, `Invalid ${property} - ${error.message}`) + } + return next() + } +} + +export function validateBody(schema: AnyZodObject) { + return validate(schema, "body") +} + +// export function validateParams(schema: Joi.Schema) { +// return validate(schema, "params") +// } From 36410aac0bff05ff4f507613c1003c9268e3a036 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 4 Oct 2024 11:45:18 +0200 Subject: [PATCH 003/192] Display readable errors --- packages/server/package.json | 6 +- .../server/src/middleware/zod-validator.ts | 6 +- yarn.lock | 270 ++++++++---------- 3 files changed, 129 insertions(+), 153 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 76dd03b5a8..15e78fe7a8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -119,7 +119,8 @@ "uuid": "^8.3.2", "validate.js": "0.13.1", "worker-farm": "1.7.0", - "xml2js": "0.5.0" + "xml2js": "0.5.0", + "zod-validation-error": "^3.4.0" }, "devDependencies": { "@babel/preset-env": "7.16.11", @@ -158,7 +159,8 @@ "tsconfig-paths": "4.0.0", "typescript": "5.5.2", "update-dotenv": "1.1.1", - "yargs": "13.2.4" + "yargs": "13.2.4", + "zod": "^3.23.8" }, "nx": { "targets": { diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts index 6fb338e5e8..7b51d908c1 100644 --- a/packages/server/src/middleware/zod-validator.ts +++ b/packages/server/src/middleware/zod-validator.ts @@ -1,6 +1,8 @@ -import { AnyZodObject } from "zod" import { Ctx } from "@budibase/types" +import { AnyZodObject } from "zod" +import { fromZodError } from "zod-validation-error" + function validate(schema: AnyZodObject, property: "body" | "params") { // Return a Koa middleware function return (ctx: Ctx, next: any) => { @@ -18,7 +20,7 @@ function validate(schema: AnyZodObject, property: "body" | "params") { const { error } = schema.safeParse(params) if (error) { - ctx.throw(400, `Invalid ${property} - ${error.message}`) + ctx.throw(400, fromZodError(error)) } return next() } diff --git a/yarn.lock b/yarn.lock index bc18155a9d..9b258a2ba0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -764,20 +764,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@^3.4.1": - version "3.4.2" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" - integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== +"@azure/identity@4.2.1", "@azure/identity@^3.4.1": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" + integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.6.1" + "@azure/core-util" "^1.3.0" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.5.0" - "@azure/msal-node" "^2.5.1" + "@azure/msal-browser" "^3.11.1" + "@azure/msal-node" "^2.9.2" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -808,7 +808,7 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.5.0": +"@azure/msal-browser@^3.11.1": version "3.25.0" resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.25.0.tgz#7ce0949977bc9e0c58319f7090c44fe5537104d4" integrity sha512-a0Y7pmSy8SC1s9bvwr+REvyAA1nQcITlZvkElM2gNUPYFTTNUTEdcpg73TmawNucyMdZ9xb/GFcuhrLOqYAzwg== @@ -820,7 +820,7 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.15.0.tgz#0e27ac0bb88fe100f4f8d1605b64d5c268636a55" integrity sha512-ImAQHxmpMneJ/4S8BRFhjt1MZ3bppmpRPYYNyzeQPeFN288YKbb8TmmISQEbtfkQ1BPASvYZU5doIZOPBAqENQ== -"@azure/msal-node@^2.5.1": +"@azure/msal-node@^2.9.2": version "2.15.0" resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.15.0.tgz#50bf8e692a6656027c073a75d877a8a478aafdfd" integrity sha512-gVPW8YLz92ZeCibQH2QUw96odJoiM3k/ZPH3f2HxptozmH6+OnyyvKXo/Egg39HAM230akarQKHf0W74UHlh0Q== @@ -2051,6 +2051,44 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/backend-core@2.32.12": + version "0.0.0" + dependencies: + "@budibase/nano" "10.1.5" + "@budibase/pouchdb-replication-stream" "1.2.11" + "@budibase/shared-core" "0.0.0" + "@budibase/types" "0.0.0" + aws-cloudfront-sign "3.0.2" + aws-sdk "2.1030.0" + bcrypt "5.1.0" + bcryptjs "2.4.3" + bull "4.10.1" + correlation-id "4.0.0" + dd-trace "5.2.0" + dotenv "16.0.1" + ioredis "5.3.2" + joi "17.6.0" + jsonwebtoken "9.0.2" + knex "2.4.2" + koa-passport "^6.0.0" + koa-pino-logger "4.0.0" + lodash "4.17.21" + node-fetch "2.6.7" + passport-google-oauth "2.0.0" + passport-local "1.0.0" + passport-oauth2-refresh "^2.1.0" + pino "8.11.0" + pino-http "8.3.3" + posthog-node "4.0.1" + pouchdb "7.3.0" + pouchdb-find "7.2.2" + redlock "4.2.0" + rotating-file-stream "3.1.0" + sanitize-s3-objectkey "0.0.1" + semver "^7.5.4" + tar-fs "2.1.1" + uuid "^8.3.2" + "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2093,6 +2131,48 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "2.32.12" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.12.tgz#a5a975ad223e40eb186576f9982e9bef583da906" + integrity sha512-18BeOwMKAkjrgIeyvXTnUc07pcFKdEg0aYqoOxkGsz182YUEk1r+unalvSSBW8hsKObZw1pkcx45dAFfI14wLA== + dependencies: + "@anthropic-ai/sdk" "^0.27.3" + "@budibase/backend-core" "2.32.12" + "@budibase/shared-core" "2.32.12" + "@budibase/string-templates" "2.32.12" + "@budibase/types" "2.32.12" + "@koa/router" "8.0.8" + bull "4.10.1" + dd-trace "5.2.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + openai "4.59.0" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + +"@budibase/shared-core@2.32.12": + version "0.0.0" + dependencies: + "@budibase/types" "0.0.0" + cron-validate "1.4.5" + +"@budibase/string-templates@2.32.12": + version "0.0.0" + dependencies: + "@budibase/handlebars-helpers" "^0.13.2" + dayjs "^1.10.8" + handlebars "^4.7.8" + lodash.clonedeep "^4.5.0" + +"@budibase/types@2.32.12": + version "0.0.0" + dependencies: + scim-patch "^0.8.1" + zod "^3.23.8" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7247,23 +7327,7 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11063,14 +11127,7 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5: - version "4.2.5" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" - integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== - dependencies: - strnum "^1.0.5" - -fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11319,11 +11376,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0: - version "1.15.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" - integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== - follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12406,12 +12458,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -12931,11 +12978,6 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== - is-builtin-module@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" @@ -13412,6 +13454,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14316,14 +14363,7 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@^3.0.2, kind-of@^3.1.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -15943,7 +15983,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16137,25 +16177,13 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9, node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17291,15 +17319,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18559,7 +18579,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19594,11 +19614,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -19671,33 +19686,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21276,7 +21271,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21286,14 +21281,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21809,6 +21796,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22592,33 +22587,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22628,11 +22600,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" @@ -22881,6 +22848,11 @@ zip-stream@^6.0.1: compress-commons "^6.0.2" readable-stream "^4.0.0" +zod-validation-error@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/zod-validation-error/-/zod-validation-error-3.4.0.tgz#3a8a1f55c65579822d7faa190b51336c61bee2a6" + integrity sha512-ZOPR9SVY6Pb2qqO5XHt+MkkTRxGXb4EVtnjc9JpXUOtUB1T9Ru7mZOT361AN3MsetVe7R0a1KZshJDZdgp9miQ== + zod@^3.23.8: version "3.23.8" resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" From eea9eade5c0eedfeb578a9b688a3654ec93e7280 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 8 Oct 2024 12:40:09 +0200 Subject: [PATCH 004/192] Start types --- .../server/src/api/controllers/row/index.ts | 9 ++++--- packages/types/src/api/web/app/rows.ts | 27 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 680a7671d5..769a1dbe5e 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -17,6 +17,7 @@ import { GetRowResponse, PatchRowRequest, PatchRowResponse, + RequiredKeys, Row, RowAttachment, RowSearchParams, @@ -211,15 +212,17 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) + const searchRequest = ctx.request.body + const enrichedQuery = await utils.enrichSearchContext( - { ...ctx.request.body.query }, + { ...searchRequest.query }, { user: sdk.users.getUserContextBindings(ctx.user), } ) - const searchParams: RowSearchParams = { - ...ctx.request.body, + const searchParams: RequiredKeys = { + ...searchRequest, query: enrichedQuery, tableId, viewId, diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index ce6f6f672d..166c6965c4 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,7 +1,8 @@ import { SearchFilters, RowSearchParams } from "../../../sdk" import { Row } from "../../../documents" -import { PaginationResponse, SortOrder } from "../../../api" +import { PaginationResponse, SortOrder, SortType } from "../../../api" import { ReadStream } from "fs" +import { z } from "zod" export interface SaveRowRequest extends Row {} @@ -13,7 +14,29 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} -export interface SearchRowRequest extends Omit {} +const searchRowRequest = z.object({ + table: z.string(), + query: z.object({ + allOr: z.boolean().optional(), + }), + paginate: z.boolean().optional(), + bookmark: z.union([z.string(), z.number()]).optional(), + limit: z.number().optional(), + sort: z.string().optional(), + sortOrder: z.nativeEnum(SortOrder).optional(), + sortType: z.nativeEnum(SortType).optional(), + version: z.string().optional(), + disableEscaping: z.boolean().optional(), + countRows: z.boolean().optional(), + + // viewId?: string + // query?: SearchFilters + + // fields?: string[] + // indexer?: () => Promise + // rows?: Row[] +}) +export type SearchRowRequest = z.infer export interface SearchViewRowRequest extends Pick< From d1a69a66607e43ca7ec80c308f474deef65de8d8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 11:40:29 +0200 Subject: [PATCH 005/192] Use validation --- packages/server/src/api/routes/row.ts | 3 +++ packages/server/src/middleware/zod-validator.ts | 2 +- packages/types/src/api/web/app/rows.ts | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index e443b2daeb..fc09b8c81d 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -5,6 +5,8 @@ import { paramResource, paramSubResource } from "../../middleware/resourceId" import { permissions } from "@budibase/backend-core" import { internalSearchValidator } from "./utils/validators" import trimViewRowInfo from "../../middleware/trimViewRowInfo" +import { validateBody } from "../../middleware/zod-validator" +import { searchRowRequestValidator } from "@budibase/types" const { PermissionType, PermissionLevel } = permissions @@ -32,6 +34,7 @@ router .post( "/api/:sourceId/search", internalSearchValidator(), + validateBody(searchRowRequestValidator), paramResource("sourceId"), authorized(PermissionType.TABLE, PermissionLevel.READ), rowController.search diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts index 7b51d908c1..6611efbeb1 100644 --- a/packages/server/src/middleware/zod-validator.ts +++ b/packages/server/src/middleware/zod-validator.ts @@ -30,6 +30,6 @@ export function validateBody(schema: AnyZodObject) { return validate(schema, "body") } -// export function validateParams(schema: Joi.Schema) { +// export function validateParams(schema: AnyZodObject) { // return validate(schema, "params") // } diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 166c6965c4..e696430eb4 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -15,7 +15,6 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} const searchRowRequest = z.object({ - table: z.string(), query: z.object({ allOr: z.boolean().optional(), }), @@ -36,6 +35,7 @@ const searchRowRequest = z.object({ // indexer?: () => Promise // rows?: Row[] }) +export const searchRowRequestValidator = searchRowRequest export type SearchRowRequest = z.infer export interface SearchViewRowRequest From 76f155f31124656a9bba6c41c2fb995ed526655c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 12:31:10 +0200 Subject: [PATCH 006/192] Fix jest import issues --- packages/types/src/api/web/app/rows.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index e696430eb4..3aee8e754e 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,6 +1,10 @@ -import { SearchFilters, RowSearchParams } from "../../../sdk" +import { SearchFilters } from "../../../sdk" import { Row } from "../../../documents" -import { PaginationResponse, SortOrder, SortType } from "../../../api" +import { + PaginationResponse, + SortOrder, + SortType, +} from "../../../api/web/pagination" import { ReadStream } from "fs" import { z } from "zod" From b6731494e7138189d5dc36f3c2feeeccf033e303 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 12:39:14 +0200 Subject: [PATCH 007/192] Trim object --- packages/server/src/middleware/zod-validator.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts index 6611efbeb1..f990ee0642 100644 --- a/packages/server/src/middleware/zod-validator.ts +++ b/packages/server/src/middleware/zod-validator.ts @@ -10,18 +10,25 @@ function validate(schema: AnyZodObject, property: "body" | "params") { return next() } let params = null + let setClean: ((data: any) => void) | undefined if (ctx[property] != null) { params = ctx[property] + setClean = data => (ctx[property] = data) } else if (property === "body" && ctx.request[property] != null) { params = ctx.request[property] - } else if (ctx.request.get(property) != null) { - params = ctx.request.get(property) + setClean = data => (ctx.request[property] = data) + } else if (property === "params") { + params = ctx.request.query + setClean = data => (ctx.request.query = data) } - const { error } = schema.safeParse(params) - if (error) { - ctx.throw(400, fromZodError(error)) + const result = schema.safeParse(params) + if (!result.success) { + ctx.throw(400, fromZodError(result.error)) + } else { + setClean?.(result.data) } + return next() } } From 7b789e785369ff3ed121b677fb3340a75ba798f0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 13:11:32 +0200 Subject: [PATCH 008/192] Type query --- packages/types/src/api/web/app/rows.ts | 43 +++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 3aee8e754e..45e065cc59 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,4 +1,11 @@ -import { SearchFilters } from "../../../sdk" +import { + ArrayOperator, + BasicOperator, + LogicalOperator, + RangeOperator, + SearchFilterKey, + SearchFilters, +} from "../../../sdk" import { Row } from "../../../documents" import { PaginationResponse, @@ -18,9 +25,43 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} +// TODO: exclude InternalSearchFilterOperator.COMPLEX_ID_OPERATOR +const stringBasicFilter = z.record(z.string(), z.string()) +const basicFilter = z.record(z.string(), z.any()) +const arrayFilter = z.any() +const logicFilter = z.any() + +const stringOrNumber = z.union([z.string(), z.number()]) + +const queryFilterValidation: Record = { + [BasicOperator.STRING]: stringBasicFilter.optional(), + [BasicOperator.FUZZY]: stringBasicFilter.optional(), + [RangeOperator.RANGE]: z + .record( + z.string(), + z.union([ + z.object({ high: stringOrNumber, low: stringOrNumber }), + z.object({ high: stringOrNumber }), + z.object({ low: stringOrNumber }), + ]) + ) + .optional(), + [BasicOperator.EQUAL]: basicFilter.optional(), + [BasicOperator.NOT_EQUAL]: basicFilter.optional(), + [BasicOperator.EMPTY]: basicFilter.optional(), + [BasicOperator.NOT_EMPTY]: basicFilter.optional(), + [ArrayOperator.ONE_OF]: arrayFilter.optional(), + [ArrayOperator.CONTAINS]: arrayFilter.optional(), + [ArrayOperator.NOT_CONTAINS]: arrayFilter.optional(), + [ArrayOperator.CONTAINS_ANY]: arrayFilter.optional(), + [LogicalOperator.AND]: logicFilter.optional(), + [LogicalOperator.OR]: logicFilter.optional(), +} + const searchRowRequest = z.object({ query: z.object({ allOr: z.boolean().optional(), + ...queryFilterValidation, }), paginate: z.boolean().optional(), bookmark: z.union([z.string(), z.number()]).optional(), From 4324d9683306eb1667c0948313f39d8c5d83118d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 13:20:20 +0200 Subject: [PATCH 009/192] Add onEmptyFilter --- packages/types/src/api/web/app/rows.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 45e065cc59..4652ea7df5 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -1,6 +1,7 @@ import { ArrayOperator, BasicOperator, + EmptyFilterOption, LogicalOperator, RangeOperator, SearchFilterKey, @@ -59,10 +60,13 @@ const queryFilterValidation: Record = { } const searchRowRequest = z.object({ - query: z.object({ - allOr: z.boolean().optional(), - ...queryFilterValidation, - }), + query: z + .object({ + allOr: z.boolean().optional(), + onEmptyFilter: z.nativeEnum(EmptyFilterOption).optional(), + ...queryFilterValidation, + }) + .optional(), paginate: z.boolean().optional(), bookmark: z.union([z.string(), z.number()]).optional(), limit: z.number().optional(), From e6b3b4f2a1ef732460b850dd3f5afdcb22a88e6d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 13:33:27 +0200 Subject: [PATCH 010/192] Type array filter --- packages/types/src/api/web/app/rows.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 4652ea7df5..6cc5e575a6 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -29,7 +29,7 @@ export interface PatchRowResponse extends Row {} // TODO: exclude InternalSearchFilterOperator.COMPLEX_ID_OPERATOR const stringBasicFilter = z.record(z.string(), z.string()) const basicFilter = z.record(z.string(), z.any()) -const arrayFilter = z.any() +const arrayFilter = z.record(z.string(), z.union([z.any().array(), z.string()])) const logicFilter = z.any() const stringOrNumber = z.union([z.string(), z.number()]) From e095a596a151582d1070bd661a4579bfb491d0d8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 13:40:39 +0200 Subject: [PATCH 011/192] Type logic filter --- packages/types/src/api/web/app/rows.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 6cc5e575a6..a02081366c 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -30,7 +30,11 @@ export interface PatchRowResponse extends Row {} const stringBasicFilter = z.record(z.string(), z.string()) const basicFilter = z.record(z.string(), z.any()) const arrayFilter = z.record(z.string(), z.union([z.any().array(), z.string()])) -const logicFilter = z.any() +const logicFilter = z.lazy(() => + z.object({ + conditions: z.array(z.object(queryFilterValidation)), + }) +) const stringOrNumber = z.union([z.string(), z.number()]) From 60a5f4a2e72da9130cedb32c5ed4948d3eb2934b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 15:11:11 +0200 Subject: [PATCH 012/192] Types --- packages/server/src/api/controllers/row/index.ts | 13 ++++++++++++- packages/server/src/api/routes/tests/search.spec.ts | 1 + packages/types/src/api/web/app/rows.ts | 7 ------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 769a1dbe5e..0e4e305ce7 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -222,10 +222,21 @@ export async function search(ctx: Ctx) { ) const searchParams: RequiredKeys = { - ...searchRequest, query: enrichedQuery, tableId, viewId, + bookmark: searchRequest.bookmark, + paginate: searchRequest.paginate, + limit: searchRequest.limit, + sort: searchRequest.sort, + sortOrder: searchRequest.sortOrder, + sortType: searchRequest.sortType, + countRows: searchRequest.countRows, + version: searchRequest.version, + disableEscaping: searchRequest.disableEscaping, + fields: undefined, + indexer: undefined, + rows: undefined, } ctx.status = 200 diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ccc9bfdc9..a53f650d22 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -187,6 +187,7 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), { ...this.query, + query: this.query.query!, }) } else { return config.api.row.search(tableOrViewId, this.query) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index a02081366c..013e349fe5 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -80,13 +80,6 @@ const searchRowRequest = z.object({ version: z.string().optional(), disableEscaping: z.boolean().optional(), countRows: z.boolean().optional(), - - // viewId?: string - // query?: SearchFilters - - // fields?: string[] - // indexer?: () => Promise - // rows?: Row[] }) export const searchRowRequestValidator = searchRowRequest export type SearchRowRequest = z.infer From 5cad93ff75eb1eb02669c1edb35e90457fa4e29f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 10 Oct 2024 15:15:54 +0200 Subject: [PATCH 013/192] Fix --- packages/server/src/api/controllers/row/views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 622688deb6..9ceb14ed2c 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -35,7 +35,7 @@ export async function searchView( > = { tableId: view.tableId, viewId: view.id, - query: body.query, + query: body.query || {}, fields: viewFields, ...getSortOptions(body, view), limit: body.limit, From 6edaf5e160dbd6cbabc76c30df3ab2f83d9a9772 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 4 Nov 2024 13:33:12 +0000 Subject: [PATCH 014/192] Making sure all protected columns are kept during row save/patch operations. --- .../server/src/utilities/rowProcessor/index.ts | 14 ++++++++++---- packages/shared-core/src/constants/rows.ts | 4 ++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 910e9d220f..331d868dc7 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -31,6 +31,8 @@ import { import { isExternalTableID } from "../../integrations/utils" import { helpers, + isExternalColumnName, + isInternalColumnName, PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" @@ -202,14 +204,18 @@ export async function inputProcessing( const clonedRow = cloneDeep(row) const table = await getTableFromSource(source) - const dontCleanseKeys = ["type", "_id", "_rev", "tableId"] for (const [key, value] of Object.entries(clonedRow)) { const field = table.schema[key] // cleanse fields that aren't in the schema + if ( + !field && isExternalTableID(table._id!) + ? !isExternalColumnName(key) + : !isInternalColumnName(key) + ) { + delete clonedRow[key] + } + // field isn't found - might be a built-in column, skip over it if (!field) { - if (dontCleanseKeys.indexOf(key) === -1) { - delete clonedRow[key] - } continue } // remove any formula values, they are to be generated diff --git a/packages/shared-core/src/constants/rows.ts b/packages/shared-core/src/constants/rows.ts index 03663a4014..23c123aaf4 100644 --- a/packages/shared-core/src/constants/rows.ts +++ b/packages/shared-core/src/constants/rows.ts @@ -12,3 +12,7 @@ export const PROTECTED_EXTERNAL_COLUMNS = ["_id", "_rev", "tableId"] as const export function isInternalColumnName(name: string): boolean { return (PROTECTED_INTERNAL_COLUMNS as readonly string[]).includes(name) } + +export function isExternalColumnName(name: string): boolean { + return (PROTECTED_EXTERNAL_COLUMNS as readonly string[]).includes(name) +} From 6bb668ee33fd84fb12b98c9120725ca718c43654 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 4 Nov 2024 15:03:09 +0100 Subject: [PATCH 015/192] Fix tests --- packages/server/src/api/routes/tests/row.spec.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 8c4a01b07b..9213967d38 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2799,6 +2799,8 @@ describe.each([ name: "foo", description: "bar", tableId, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }) }) @@ -2820,6 +2822,8 @@ describe.each([ id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }) }) @@ -2840,6 +2844,8 @@ describe.each([ _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }) }) @@ -2917,6 +2923,8 @@ describe.each([ id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }) }) @@ -2941,6 +2949,8 @@ describe.each([ _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }) }) From 8519cfc495b281dd6fbc588d4496ec9bf9852ecc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 14:05:57 +0100 Subject: [PATCH 016/192] Fix typing --- packages/server/src/api/controllers/row/utils/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 4188fcced3..5b60143792 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -175,7 +175,7 @@ export async function enrichArrayContext( } export async function enrichSearchContext( - fields: Record, + fields: Record | undefined, inputs = {}, helpers = true ): Promise> { From 3250ad49476c81520c5039240e535b17dc0b25ad Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 14:15:46 +0100 Subject: [PATCH 017/192] Allow nullable bookmarks --- .../src/api/routes/tests/search.spec.ts | 21 ++++++++++++++++--- packages/types/src/api/web/app/rows.ts | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index fe13e5311f..6600cf0da1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -24,6 +24,7 @@ import { JsonFieldSubType, LogicalOperator, RelationshipType, + RequiredKeys, Row, RowSearchParams, SearchFilters, @@ -206,9 +207,23 @@ datasourceDescribe( private async performSearch(): Promise> { if (isInMemory) { - return dataFilters.search(_.cloneDeep(rows), { - ...this.query, - }) + const query: RequiredKeys> = { + sort: this.query.sort, + query: this.query.query || {}, + paginate: this.query.paginate, + bookmark: this.query.bookmark, + limit: this.query.limit, + sortOrder: this.query.sortOrder, + sortType: this.query.sortType, + version: this.query.version, + disableEscaping: this.query.disableEscaping, + countRows: this.query.countRows, + viewId: undefined, + fields: undefined, + indexer: undefined, + rows: undefined, + } + return dataFilters.search(_.cloneDeep(rows), query) } else { return config.api.row.search(tableOrViewId, this.query) } diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 013e349fe5..34ef1ee4c1 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -72,7 +72,7 @@ const searchRowRequest = z.object({ }) .optional(), paginate: z.boolean().optional(), - bookmark: z.union([z.string(), z.number()]).optional(), + bookmark: z.union([z.string(), z.number()]).nullable().optional(), limit: z.number().optional(), sort: z.string().optional(), sortOrder: z.nativeEnum(SortOrder).optional(), From cf7daf06027a9811e647cd06f33b123cdffbc3ed Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 14:16:10 +0100 Subject: [PATCH 018/192] Fix --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6600cf0da1..2c13cac014 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -211,7 +211,7 @@ datasourceDescribe( sort: this.query.sort, query: this.query.query || {}, paginate: this.query.paginate, - bookmark: this.query.bookmark, + bookmark: this.query.bookmark ?? undefined, limit: this.query.limit, sortOrder: this.query.sortOrder, sortType: this.query.sortType, From 414d79d73c2db01a40b4a298abcec592a8867bbd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 14:16:26 +0100 Subject: [PATCH 019/192] Fix types --- packages/server/src/api/controllers/row/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index abc751b5e4..fbaf9f8462 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -255,7 +255,7 @@ export async function search(ctx: Ctx) { query: enrichedQuery, tableId, viewId, - bookmark: searchRequest.bookmark, + bookmark: searchRequest.bookmark ?? undefined, paginate: searchRequest.paginate, limit: searchRequest.limit, sort: searchRequest.sort, From 94f475c0607a001f131da6146b3b6fe55af63a2d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 16:09:03 +0100 Subject: [PATCH 020/192] Exclude InternalSearchFilterOperator.COMPLEX_ID_OPERATOR --- packages/types/src/api/web/app/rows.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 34ef1ee4c1..520983d353 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -2,6 +2,7 @@ import { ArrayOperator, BasicOperator, EmptyFilterOption, + InternalSearchFilterOperator, LogicalOperator, RangeOperator, SearchFilterKey, @@ -26,10 +27,15 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} -// TODO: exclude InternalSearchFilterOperator.COMPLEX_ID_OPERATOR -const stringBasicFilter = z.record(z.string(), z.string()) -const basicFilter = z.record(z.string(), z.any()) -const arrayFilter = z.record(z.string(), z.union([z.any().array(), z.string()])) +const fieldKey = z + .string() + .refine(s => s !== InternalSearchFilterOperator.COMPLEX_ID_OPERATOR, { + message: `Key '${InternalSearchFilterOperator.COMPLEX_ID_OPERATOR}' is not allowed`, + }) + +const stringBasicFilter = z.record(fieldKey, z.string()) +const basicFilter = z.record(fieldKey, z.any()) +const arrayFilter = z.record(fieldKey, z.union([z.any().array(), z.string()])) const logicFilter = z.lazy(() => z.object({ conditions: z.array(z.object(queryFilterValidation)), @@ -43,7 +49,7 @@ const queryFilterValidation: Record = { [BasicOperator.FUZZY]: stringBasicFilter.optional(), [RangeOperator.RANGE]: z .record( - z.string(), + fieldKey, z.union([ z.object({ high: stringOrNumber, low: stringOrNumber }), z.object({ high: stringOrNumber }), From ed6d434843f87dfc1287217c54baa9bfe8b3976c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 16:14:03 +0100 Subject: [PATCH 021/192] Fix --- packages/server/src/api/controllers/row/views.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 7d0957b2fc..58bd40b529 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -29,19 +29,20 @@ export async function searchView( await context.ensureSnippetContext(true) - const searchOptions: RequiredKeys & - RequiredKeys< - Pick - > = { + const searchOptions: RequiredKeys = { tableId: view.tableId, viewId: view.id, query: body.query || {}, fields: viewFields, ...getSortOptions(body, view), limit: body.limit, - bookmark: body.bookmark, + bookmark: body.bookmark ?? undefined, paginate: body.paginate, countRows: body.countRows, + version: undefined, + disableEscaping: undefined, + indexer: undefined, + rows: undefined, } const result = await sdk.rows.search(searchOptions, { From 2fb4c11484cbf2a5eecdaacc6c0efba536544281 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 20 Nov 2024 18:32:02 +0100 Subject: [PATCH 022/192] Update yarn.lock --- yarn.lock | 7 ------- 1 file changed, 7 deletions(-) diff --git a/yarn.lock b/yarn.lock index aaadbbe7bd..0e7f680619 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5751,13 +5751,6 @@ dependencies: "@types/koa" "*" -"@types/koa__router@8.0.8": - version "8.0.8" - resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.8.tgz#b1e0e9a512498777d3366bbdf0e853df27ec831c" - integrity sha512-9pGCaDtzCsj4HJ8HmGuqzk8+s57sPj4njWd08GG5o92n5Xp9io2snc40CPpXFhoKcZ8OKhuu6ht4gNou9e1C2w== - dependencies: - "@types/koa" "*" - "@types/lodash@4.14.200": version "4.14.200" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149" From ebf82a0ef5dc24f42c4a735652c997cb3ca86636 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 21 Nov 2024 17:33:47 +0000 Subject: [PATCH 023/192] wip --- .../server/src/api/routes/tests/row.spec.ts | 53 +++++++++++++++++++ packages/server/src/integrations/postgres.ts | 4 ++ 2 files changed, 57 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 02995e6d0a..f0dc27708c 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3495,6 +3495,59 @@ if (descriptions.length) { ) }) }) + + !isInternal && + describe("bigint ids", () => { + let table: Table + let relatedTable: Table + + beforeAll(async () => { + const tableName = generator.guid().substring(0, 10) + await client!.schema.createTable(tableName, table => { + table.bigIncrements("id").primary() + }) + + const relatedTableName = generator.guid().substring(0, 10) + await client!.schema.createTable(relatedTableName, table => { + table.increments("id").primary() + table + .bigInteger("tableid") + .unsigned() + .references("id") + .inTable(tableName) + }) + + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource!._id!, + }) + + const tables = Object.values(resp.datasource.entities || {}) + table = tables.find(t => t.name === tableName)! + relatedTable = tables.find(t => t.name === relatedTableName)! + + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + related: { + name: "related", + type: FieldType.LINK, + tableId: relatedTable._id!, + fieldName: "tableid", + relationshipType: RelationshipType.ONE_TO_MANY, + }, + }, + }) + }) + + it.only("should be able to fetch rows with related bigint ids", async () => { + const row = await config.api.row.save(table._id!, {}) + await config.api.row.save(relatedTable._id!, { tableid: row.id }) + + const { rows } = await config.api.row.search(table._id!) + expect(rows).toEqual([]) + }) + }) } ) } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 28400f616f..881017c24e 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -39,6 +39,10 @@ if (types) { types.setTypeParser(1114, (val: any) => val) // timestamp types.setTypeParser(1082, (val: any) => val) // date types.setTypeParser(1184, (val: any) => val) // timestampz + // types.setTypeParser(114, JSON.parse) // json + // types.setTypeParser(3802, JSON.parse) // jsonb + // types.setTypeParser(199, parseJsonArray) // json[] + // types.setTypeParser(3807, parseJsonArray) // jsonb[] } const JSON_REGEX = /'{\s*.*?\s*}'::json/gs From f464a8048461c1952102950655f056409c2e5452 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 21 Nov 2024 18:09:21 +0000 Subject: [PATCH 024/192] Adding helper to create relationship to datasource API in tests. --- .../src/tests/utilities/api/datasource.ts | 66 +++++++++++++++++-- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 67484a688a..03ff6ac8d0 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -1,14 +1,17 @@ import { - Datasource, - VerifyDatasourceRequest, - CreateDatasourceResponse, - UpdateDatasourceResponse, - UpdateDatasourceRequest, - QueryJson, BuildSchemaFromSourceResponse, + CreateDatasourceResponse, + Datasource, FetchDatasourceInfoResponse, + FieldType, + QueryJson, + RelationshipType, + UpdateDatasourceRequest, + UpdateDatasourceResponse, + VerifyDatasourceRequest, } from "@budibase/types" import { Expectations, TestAPI } from "./base" +import { sql } from "@budibase/backend-core" export class DatasourceAPI extends TestAPI { create = async ( @@ -103,4 +106,55 @@ export class DatasourceAPI extends TestAPI { } ) } + + addExistingRelationship = async ( + tableId1: string, + tableId2: string, + relationshipNameInTable1: string, + relationshipNameInTable2: string, + primaryKey: string, + foreignKey: string, + expectations?: Expectations + ) => { + const tableInfo1 = sql.utils.breakExternalTableId(tableId1), + tableInfo2 = sql.utils.breakExternalTableId(tableId2) + if (tableInfo1.datasourceId !== tableInfo2.datasourceId) { + throw new Error( + "Tables are in different datasources, cannot create relationship." + ) + } + const datasource = await this.get(tableInfo1.datasourceId) + const table1 = datasource.entities?.[tableInfo1.tableName], + table2 = datasource.entities?.[tableInfo2.tableName] + if (!table1 || !table2) { + throw new Error( + "Both tables not found in datasource, cannot create relationship." + ) + } + + const table1HasPrimary = table1.primary!.includes(primaryKey) + table1.schema[relationshipNameInTable1] = { + type: FieldType.LINK, + name: relationshipNameInTable1, + tableId: tableId2, + relationshipType: table1HasPrimary + ? RelationshipType.MANY_TO_ONE + : RelationshipType.ONE_TO_MANY, + fieldName: table1HasPrimary ? foreignKey : primaryKey, + foreignKey: table1HasPrimary ? primaryKey : foreignKey, + main: table1HasPrimary, + } + table2.schema[relationshipNameInTable2] = { + type: FieldType.LINK, + name: relationshipNameInTable2, + tableId: tableId1, + relationshipType: table1HasPrimary + ? RelationshipType.ONE_TO_MANY + : RelationshipType.MANY_TO_ONE, + fieldName: table1HasPrimary ? primaryKey : foreignKey, + foreignKey: table1HasPrimary ? foreignKey : primaryKey, + main: !table1HasPrimary, + } + return await this.update(datasource, expectations) + } } From 23f29f9381e21e47f55cd073787c9f74d8686b98 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 21 Nov 2024 18:41:04 +0000 Subject: [PATCH 025/192] Adding relationship helper for defining existing relationships. --- .../server/src/api/routes/tests/row.spec.ts | 44 ++++++++------ .../src/tests/utilities/api/datasource.ts | 59 +++++++++---------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f0dc27708c..a7712a7137 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3496,18 +3496,18 @@ if (descriptions.length) { }) }) - !isInternal && + if (!isInternal) { describe("bigint ids", () => { - let table: Table - let relatedTable: Table + let table: Table, relatedTable: Table + let tableName: string, relatedTableName: string beforeAll(async () => { - const tableName = generator.guid().substring(0, 10) + tableName = generator.guid().substring(0, 10) await client!.schema.createTable(tableName, table => { table.bigIncrements("id").primary() }) - const relatedTableName = generator.guid().substring(0, 10) + relatedTableName = generator.guid().substring(0, 10) await client!.schema.createTable(relatedTableName, table => { table.increments("id").primary() table @@ -3525,17 +3525,16 @@ if (descriptions.length) { table = tables.find(t => t.name === tableName)! relatedTable = tables.find(t => t.name === relatedTableName)! - await config.api.table.save({ - ...table, - schema: { - ...table.schema, - related: { - name: "related", - type: FieldType.LINK, - tableId: relatedTable._id!, - fieldName: "tableid", - relationshipType: RelationshipType.ONE_TO_MANY, - }, + await config.api.datasource.addExistingRelationship({ + one: { + tableId: relatedTable._id!, + relationshipName: "one", + foreignKey: "tableid", + }, + many: { + tableId: table._id!, + relationshipName: "many", + primaryKey: "id", }, }) }) @@ -3545,9 +3544,20 @@ if (descriptions.length) { await config.api.row.save(relatedTable._id!, { tableid: row.id }) const { rows } = await config.api.row.search(table._id!) - expect(rows).toEqual([]) + expect(rows.length).toEqual(1) + expect(rows[0]).toEqual( + expect.objectContaining({ + many: [ + { + _id: "%5B1%5D", + primaryDisplay: 1, + }, + ], + }) + ) }) }) + } } ) } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 03ff6ac8d0..87f03c8a6f 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -108,52 +108,47 @@ export class DatasourceAPI extends TestAPI { } addExistingRelationship = async ( - tableId1: string, - tableId2: string, - relationshipNameInTable1: string, - relationshipNameInTable2: string, - primaryKey: string, - foreignKey: string, + { + one, + many, + }: { + one: { tableId: string; relationshipName: string; foreignKey: string } + many: { tableId: string; relationshipName: string; primaryKey: string } + }, expectations?: Expectations ) => { - const tableInfo1 = sql.utils.breakExternalTableId(tableId1), - tableInfo2 = sql.utils.breakExternalTableId(tableId2) - if (tableInfo1.datasourceId !== tableInfo2.datasourceId) { + const oneTableInfo = sql.utils.breakExternalTableId(one.tableId), + manyTableInfo = sql.utils.breakExternalTableId(many.tableId) + if (oneTableInfo.datasourceId !== manyTableInfo.datasourceId) { throw new Error( "Tables are in different datasources, cannot create relationship." ) } - const datasource = await this.get(tableInfo1.datasourceId) - const table1 = datasource.entities?.[tableInfo1.tableName], - table2 = datasource.entities?.[tableInfo2.tableName] - if (!table1 || !table2) { + const datasource = await this.get(oneTableInfo.datasourceId) + const oneTable = datasource.entities?.[oneTableInfo.tableName], + manyTable = datasource.entities?.[manyTableInfo.tableName] + if (!oneTable || !manyTable) { throw new Error( "Both tables not found in datasource, cannot create relationship." ) } - const table1HasPrimary = table1.primary!.includes(primaryKey) - table1.schema[relationshipNameInTable1] = { + manyTable.schema[many.relationshipName] = { type: FieldType.LINK, - name: relationshipNameInTable1, - tableId: tableId2, - relationshipType: table1HasPrimary - ? RelationshipType.MANY_TO_ONE - : RelationshipType.ONE_TO_MANY, - fieldName: table1HasPrimary ? foreignKey : primaryKey, - foreignKey: table1HasPrimary ? primaryKey : foreignKey, - main: table1HasPrimary, + name: many.relationshipName, + tableId: oneTable._id!, + relationshipType: RelationshipType.MANY_TO_ONE, + fieldName: one.foreignKey, + foreignKey: many.primaryKey, + main: true, } - table2.schema[relationshipNameInTable2] = { + oneTable.schema[one.relationshipName] = { type: FieldType.LINK, - name: relationshipNameInTable2, - tableId: tableId1, - relationshipType: table1HasPrimary - ? RelationshipType.ONE_TO_MANY - : RelationshipType.MANY_TO_ONE, - fieldName: table1HasPrimary ? primaryKey : foreignKey, - foreignKey: table1HasPrimary ? foreignKey : primaryKey, - main: !table1HasPrimary, + name: one.relationshipName, + tableId: manyTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: many.primaryKey, + foreignKey: one.foreignKey, } return await this.update(datasource, expectations) } From e25d92a49e759e06e20e8611463a20f994323d95 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:18:12 +0000 Subject: [PATCH 026/192] wip --- packages/backend-core/src/sql/sql.ts | 23 +++++++++++++++---- .../server/src/api/routes/tests/row.spec.ts | 14 ++++++++++- packages/server/src/integrations/postgres.ts | 4 ---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e4b2b843af..4ec9246cbb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1191,8 +1191,9 @@ class InternalBuilder { return withSchema } - private buildJsonField(field: string): string { + private buildJsonField(table: Table, field: string): string { const parts = field.split(".") + const baseName = parts[parts.length - 1] let unaliased: string let tableField: string @@ -1205,10 +1206,22 @@ class InternalBuilder { tableField = unaliased } + const schema = table.schema[baseName] + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," - return this.knex - .raw(`?${separator}??`, [unaliased, this.rawQuotedIdentifier(tableField)]) - .toString() + let identifier = this.rawQuotedIdentifier(tableField) + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } else if (schema.type === FieldType.LINK) { + const otherTable = this.query.meta.tables![schema.tableId] + const otherField = otherTable.schema[schema.fieldName] + if (otherField.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + } else if (schema.autocolumn && schema.autoReason === "foreign_key") + return this.knex + .raw(`?${separator}??`, [unaliased, identifier]) + .toString() } maxFunctionParameters() { @@ -1272,7 +1285,7 @@ class InternalBuilder { Math.floor(this.maxFunctionParameters() / 2) ) const fieldList: string = relationshipFields - .map(field => this.buildJsonField(field)) + .map(field => this.buildJsonField(relatedTable!, field)) .join(",") // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f0dc27708c..0ad5439653 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3545,7 +3545,19 @@ if (descriptions.length) { await config.api.row.save(relatedTable._id!, { tableid: row.id }) const { rows } = await config.api.row.search(table._id!) - expect(rows).toEqual([]) + expect(rows).toEqual([ + expect.objectContaining({ + _id: "%5B'1'%5D", + _rev: "rev", + id: "1", + related: [ + { + _id: "%5B'1'%5D", + primaryDisplay: 1, + }, + ], + }), + ]) }) }) } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 881017c24e..28400f616f 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -39,10 +39,6 @@ if (types) { types.setTypeParser(1114, (val: any) => val) // timestamp types.setTypeParser(1082, (val: any) => val) // date types.setTypeParser(1184, (val: any) => val) // timestampz - // types.setTypeParser(114, JSON.parse) // json - // types.setTypeParser(3802, JSON.parse) // jsonb - // types.setTypeParser(199, parseJsonArray) // json[] - // types.setTypeParser(3807, parseJsonArray) // jsonb[] } const JSON_REGEX = /'{\s*.*?\s*}'::json/gs From 93d8bd792c158fdaf6ed2630edb03088135fb483 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:36:18 +0000 Subject: [PATCH 027/192] Trying to make it easier to understand my tests. --- packages/backend-core/src/sql/sql.ts | 6 +-- .../server/src/api/routes/tests/row.spec.ts | 44 +++++++++---------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4ec9246cbb..b8aaca0f71 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1218,10 +1218,8 @@ class InternalBuilder { if (otherField.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } - } else if (schema.autocolumn && schema.autoReason === "foreign_key") - return this.knex - .raw(`?${separator}??`, [unaliased, identifier]) - .toString() + } + return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 0752d38f17..a9d56c4577 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table: Table, relatedTable: Table - let tableName: string, relatedTableName: string + let table1: Table, table2: Table + let table1Name: string, table2Name: string beforeAll(async () => { - tableName = generator.guid().substring(0, 10) - await client!.schema.createTable(tableName, table => { - table.bigIncrements("id").primary() + table1Name = `table1-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table1Name, table => { + table.bigIncrements("table1Id").primary() }) - relatedTableName = generator.guid().substring(0, 10) - await client!.schema.createTable(relatedTableName, table => { - table.increments("id").primary() + table2Name = `table2-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table2Name, table => { + table.increments("table2Id").primary() table - .bigInteger("tableid") + .bigInteger("table1Ref") .unsigned() - .references("id") - .inTable(tableName) + .references("table1Id") + .inTable(table1Name) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,34 +3522,34 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table = tables.find(t => t.name === tableName)! - relatedTable = tables.find(t => t.name === relatedTableName)! + table1 = tables.find(t => t.name === table1Name)! + table2 = tables.find(t => t.name === table2Name)! await config.api.datasource.addExistingRelationship({ one: { - tableId: relatedTable._id!, + tableId: table2._id!, relationshipName: "one", - foreignKey: "tableid", + foreignKey: "table1Ref", }, many: { - tableId: table._id!, + tableId: table1._id!, relationshipName: "many", - primaryKey: "id", + primaryKey: "table1Id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table._id!, {}) - await config.api.row.save(relatedTable._id!, { tableid: row.id }) + const row = await config.api.row.save(table1._id!, {}) + await config.api.row.save(table2._id!, { table1Ref: row.id }) - const { rows } = await config.api.row.search(table._id!) + const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - id: "1", - related: [ + table1Id: "1", + many: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From d1b0bf965b15cee6e57c770e13a065c94496258a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:51:47 +0000 Subject: [PATCH 028/192] wip --- packages/backend-core/src/sql/sql.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b8aaca0f71..d7d7235f3a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,15 +1210,15 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } else if (schema.type === FieldType.LINK) { - const otherTable = this.query.meta.tables![schema.tableId] - const otherField = otherTable.schema[schema.fieldName] - if (otherField.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } - } + // if (schema.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } else if (schema.type === FieldType.LINK) { + // const otherTable = this.query.meta.tables![schema.tableId] + // const otherField = otherTable.schema[schema.fieldName] + // if (otherField.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } + // } return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } From 50a7539e2589481cc35a7043327403a5b63ebf18 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:03 +0000 Subject: [PATCH 029/192] Revert "wip" This reverts commit d1b0bf965b15cee6e57c770e13a065c94496258a. --- packages/backend-core/src/sql/sql.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index d7d7235f3a..b8aaca0f71 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,15 +1210,15 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - // if (schema.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } else if (schema.type === FieldType.LINK) { - // const otherTable = this.query.meta.tables![schema.tableId] - // const otherField = otherTable.schema[schema.fieldName] - // if (otherField.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } - // } + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } else if (schema.type === FieldType.LINK) { + const otherTable = this.query.meta.tables![schema.tableId] + const otherField = otherTable.schema[schema.fieldName] + if (otherField.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + } return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } From c3b2c2e1c7400b9c43f1ab1d252aa42e81425136 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:11 +0000 Subject: [PATCH 030/192] Revert "Trying to make it easier to understand my tests." This reverts commit 93d8bd792c158fdaf6ed2630edb03088135fb483. --- packages/backend-core/src/sql/sql.ts | 6 ++- .../server/src/api/routes/tests/row.spec.ts | 44 +++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b8aaca0f71..4ec9246cbb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1218,8 +1218,10 @@ class InternalBuilder { if (otherField.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } - } - return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() + } else if (schema.autocolumn && schema.autoReason === "foreign_key") + return this.knex + .raw(`?${separator}??`, [unaliased, identifier]) + .toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index a9d56c4577..0752d38f17 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table1: Table, table2: Table - let table1Name: string, table2Name: string + let table: Table, relatedTable: Table + let tableName: string, relatedTableName: string beforeAll(async () => { - table1Name = `table1-${generator.guid().substring(0, 5)}` - await client!.schema.createTable(table1Name, table => { - table.bigIncrements("table1Id").primary() + tableName = generator.guid().substring(0, 10) + await client!.schema.createTable(tableName, table => { + table.bigIncrements("id").primary() }) - table2Name = `table2-${generator.guid().substring(0, 5)}` - await client!.schema.createTable(table2Name, table => { - table.increments("table2Id").primary() + relatedTableName = generator.guid().substring(0, 10) + await client!.schema.createTable(relatedTableName, table => { + table.increments("id").primary() table - .bigInteger("table1Ref") + .bigInteger("tableid") .unsigned() - .references("table1Id") - .inTable(table1Name) + .references("id") + .inTable(tableName) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,34 +3522,34 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table1 = tables.find(t => t.name === table1Name)! - table2 = tables.find(t => t.name === table2Name)! + table = tables.find(t => t.name === tableName)! + relatedTable = tables.find(t => t.name === relatedTableName)! await config.api.datasource.addExistingRelationship({ one: { - tableId: table2._id!, + tableId: relatedTable._id!, relationshipName: "one", - foreignKey: "table1Ref", + foreignKey: "tableid", }, many: { - tableId: table1._id!, + tableId: table._id!, relationshipName: "many", - primaryKey: "table1Id", + primaryKey: "id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { table1Ref: row.id }) + const row = await config.api.row.save(table._id!, {}) + await config.api.row.save(relatedTable._id!, { tableid: row.id }) - const { rows } = await config.api.row.search(table1._id!) + const { rows } = await config.api.row.search(table._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - table1Id: "1", - many: [ + id: "1", + related: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From 7138c02f2703b719a738de47db15fb4b75b34207 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:53:58 +0000 Subject: [PATCH 031/192] Put things back how they were. --- packages/backend-core/src/sql/sql.ts | 22 +++++++++---------- .../server/src/api/routes/tests/row.spec.ts | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4ec9246cbb..d7d7235f3a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1210,18 +1210,16 @@ class InternalBuilder { const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } else if (schema.type === FieldType.LINK) { - const otherTable = this.query.meta.tables![schema.tableId] - const otherField = otherTable.schema[schema.fieldName] - if (otherField.type === FieldType.BIGINT) { - identifier = this.castIntToString(identifier) - } - } else if (schema.autocolumn && schema.autoReason === "foreign_key") - return this.knex - .raw(`?${separator}??`, [unaliased, identifier]) - .toString() + // if (schema.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } else if (schema.type === FieldType.LINK) { + // const otherTable = this.query.meta.tables![schema.tableId] + // const otherField = otherTable.schema[schema.fieldName] + // if (otherField.type === FieldType.BIGINT) { + // identifier = this.castIntToString(identifier) + // } + // } + return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() } maxFunctionParameters() { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 0752d38f17..28542a8701 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3549,7 +3549,7 @@ if (descriptions.length) { _id: "%5B'1'%5D", _rev: "rev", id: "1", - related: [ + many: [ { _id: "%5B'1'%5D", primaryDisplay: 1, From ac3e0750bc5f55fc7d5e09b302a4207975e43315 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:56:16 +0000 Subject: [PATCH 032/192] Rename table variables. --- .../server/src/api/routes/tests/row.spec.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 28542a8701..56efb2980d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3498,23 +3498,23 @@ if (descriptions.length) { if (!isInternal) { describe("bigint ids", () => { - let table: Table, relatedTable: Table - let tableName: string, relatedTableName: string + let table1: Table, table2: Table + let table1Name: string, table2Name: string beforeAll(async () => { - tableName = generator.guid().substring(0, 10) - await client!.schema.createTable(tableName, table => { + table1Name = `table1-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table1Name, table => { table.bigIncrements("id").primary() }) - relatedTableName = generator.guid().substring(0, 10) - await client!.schema.createTable(relatedTableName, table => { + table2Name = `table2-${generator.guid().substring(0, 5)}` + await client!.schema.createTable(table2Name, table => { table.increments("id").primary() table .bigInteger("tableid") .unsigned() .references("id") - .inTable(tableName) + .inTable(table1Name) }) const resp = await config.api.datasource.fetchSchema({ @@ -3522,17 +3522,17 @@ if (descriptions.length) { }) const tables = Object.values(resp.datasource.entities || {}) - table = tables.find(t => t.name === tableName)! - relatedTable = tables.find(t => t.name === relatedTableName)! + table1 = tables.find(t => t.name === table1Name)! + table2 = tables.find(t => t.name === table2Name)! await config.api.datasource.addExistingRelationship({ one: { - tableId: relatedTable._id!, + tableId: table2._id!, relationshipName: "one", foreignKey: "tableid", }, many: { - tableId: table._id!, + tableId: table1._id!, relationshipName: "many", primaryKey: "id", }, @@ -3540,10 +3540,10 @@ if (descriptions.length) { }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table._id!, {}) - await config.api.row.save(relatedTable._id!, { tableid: row.id }) + const row = await config.api.row.save(table1._id!, {}) + await config.api.row.save(table2._id!, { tableid: row.id }) - const { rows } = await config.api.row.search(table._id!) + const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", From 1c27ddcb8abbbc9247746552f6deab7c28a9254d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 17:58:40 +0000 Subject: [PATCH 033/192] rename table fields --- packages/server/src/api/routes/tests/row.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 56efb2980d..8809ae5cbc 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3504,16 +3504,16 @@ if (descriptions.length) { beforeAll(async () => { table1Name = `table1-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table1Name, table => { - table.bigIncrements("id").primary() + table.bigIncrements("table1Id").primary() }) table2Name = `table2-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table2Name, table => { - table.increments("id").primary() + table.increments("table2Id").primary() table - .bigInteger("tableid") + .bigInteger("table1Ref") .unsigned() - .references("id") + .references("table1Id") .inTable(table1Name) }) @@ -3529,26 +3529,26 @@ if (descriptions.length) { one: { tableId: table2._id!, relationshipName: "one", - foreignKey: "tableid", + foreignKey: "table1Ref", }, many: { tableId: table1._id!, relationshipName: "many", - primaryKey: "id", + primaryKey: "table1Id", }, }) }) it.only("should be able to fetch rows with related bigint ids", async () => { const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { tableid: row.id }) + await config.api.row.save(table2._id!, { table1Ref: row.table1Id }) const { rows } = await config.api.row.search(table1._id!) expect(rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", _rev: "rev", - id: "1", + table1Id: "1", many: [ { _id: "%5B'1'%5D", From 26fa0222a18d70978f7cf51404974d0b1fdfd791 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 12:00:30 +0000 Subject: [PATCH 034/192] Get all bigint tests passing. --- packages/backend-core/src/sql/sql.ts | 46 +++++++++++-------- .../server/src/api/routes/tests/row.spec.ts | 42 ++++++++++++----- .../src/integrations/microsoftSqlServer.ts | 3 +- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index d7d7235f3a..e97e31869c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1191,7 +1191,7 @@ class InternalBuilder { return withSchema } - private buildJsonField(table: Table, field: string): string { + private buildJsonField(table: Table, field: string): [string, Knex.Raw] { const parts = field.split(".") const baseName = parts[parts.length - 1] let unaliased: string @@ -1208,18 +1208,11 @@ class InternalBuilder { const schema = table.schema[baseName] - const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," let identifier = this.rawQuotedIdentifier(tableField) - // if (schema.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } else if (schema.type === FieldType.LINK) { - // const otherTable = this.query.meta.tables![schema.tableId] - // const otherField = otherTable.schema[schema.fieldName] - // if (otherField.type === FieldType.BIGINT) { - // identifier = this.castIntToString(identifier) - // } - // } - return this.knex.raw(`?${separator}??`, [unaliased, identifier]).toString() + if (schema.type === FieldType.BIGINT) { + identifier = this.castIntToString(identifier) + } + return [unaliased, identifier] } maxFunctionParameters() { @@ -1282,8 +1275,14 @@ class InternalBuilder { 0, Math.floor(this.maxFunctionParameters() / 2) ) - const fieldList: string = relationshipFields - .map(field => this.buildJsonField(relatedTable!, field)) + const fieldList = relationshipFields.map(field => + this.buildJsonField(relatedTable!, field) + ) + const fieldListFormatted = fieldList + .map(f => { + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," + return this.knex.raw(`?${separator}??`, [f[0], f[1]]).toString() + }) .join(",") // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over @@ -1331,35 +1330,42 @@ class InternalBuilder { // need to check the junction table document is to the right column, this is just for SQS subQuery = this.addJoinFieldCheck(subQuery, relationship) wrapperQuery = standardWrap( - this.knex.raw(`json_group_array(json_object(${fieldList}))`) + this.knex.raw( + `json_group_array(json_object(${fieldListFormatted}))` + ) ) break case SqlClient.POSTGRES: wrapperQuery = standardWrap( - this.knex.raw(`json_agg(json_build_object(${fieldList}))`) + this.knex.raw(`json_agg(json_build_object(${fieldListFormatted}))`) ) break case SqlClient.MARIADB: // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( knex.raw( - `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})` + `json_arrayagg(json_object(${fieldListFormatted}) LIMIT ${getRelationshipLimit()})` ) ) break case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( - this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) + this.knex.raw(`json_arrayagg(json_object(${fieldListFormatted}))`) ) break case SqlClient.MS_SQL: { const comparatorQuery = knex - .select(`${fromAlias}.*`) + .select(`*`) // @ts-ignore - from alias syntax not TS supported .from({ [fromAlias]: subQuery - .select(`${toAlias}.*`) + .select( + fieldList.map(f => { + // @ts-expect-error raw is fine here, knex types are wrong + return knex.ref(f[1]).as(f[0]) + }) + ) .limit(getRelationshipLimit()), }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 8809ae5cbc..3b1b151f1f 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3496,7 +3496,7 @@ if (descriptions.length) { }) }) - if (!isInternal) { + if (!isInternal && !isOracle) { describe("bigint ids", () => { let table1: Table, table2: Table let table1Name: string, table2Name: string @@ -3504,15 +3504,14 @@ if (descriptions.length) { beforeAll(async () => { table1Name = `table1-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table1Name, table => { - table.bigIncrements("table1Id").primary() + table.bigInteger("table1Id").primary() }) table2Name = `table2-${generator.guid().substring(0, 5)}` await client!.schema.createTable(table2Name, table => { - table.increments("table2Id").primary() + table.bigInteger("table2Id").primary() table .bigInteger("table1Ref") - .unsigned() .references("table1Id") .inTable(table1Name) }) @@ -3540,21 +3539,42 @@ if (descriptions.length) { }) it.only("should be able to fetch rows with related bigint ids", async () => { - const row = await config.api.row.save(table1._id!, {}) - await config.api.row.save(table2._id!, { table1Ref: row.table1Id }) + const row = await config.api.row.save(table1._id!, { + table1Id: "1", + }) + await config.api.row.save(table2._id!, { + table2Id: "2", + table1Ref: row.table1Id, + }) - const { rows } = await config.api.row.search(table1._id!) - expect(rows).toEqual([ + let resp = await config.api.row.search(table1._id!) + expect(resp.rows).toEqual([ expect.objectContaining({ _id: "%5B'1'%5D", - _rev: "rev", table1Id: "1", many: [ { - _id: "%5B'1'%5D", - primaryDisplay: 1, + _id: "%5B'2'%5D", + primaryDisplay: "2", }, ], + tableId: table1._id, + }), + ]) + + resp = await config.api.row.search(table2._id!) + expect(resp.rows).toEqual([ + expect.objectContaining({ + _id: "%5B'2'%5D", + table2Id: "2", + table1Ref: "1", + one: [ + { + _id: "%5B'1'%5D", + primaryDisplay: "1", + }, + ], + tableId: table2._id, }), ]) }) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 1c74e6b1ff..4997f3256d 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -342,7 +342,8 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;` : query.sql this.log(sql, query.bindings) - return await request.query(sql) + const resp = await request.query(sql) + return resp } catch (err: any) { let readableMessage = getReadableErrorMessage( SourceName.SQL_SERVER, From f7c81d742c0c261ec089a5b76d915e4ecf063c8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 12:00:42 +0000 Subject: [PATCH 035/192] Remove .only --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3b1b151f1f..9e84e74d05 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3538,7 +3538,7 @@ if (descriptions.length) { }) }) - it.only("should be able to fetch rows with related bigint ids", async () => { + it("should be able to fetch rows with related bigint ids", async () => { const row = await config.api.row.save(table1._id!, { table1Id: "1", }) From b75e2e85c67e05b37ff203679e1fdb28a324c38e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 26 Nov 2024 12:03:13 +0000 Subject: [PATCH 036/192] Fixing issues with retrieval. --- packages/server/src/api/controllers/row/staticFormula.ts | 9 ++++++--- packages/server/src/utilities/rowProcessor/index.ts | 9 ++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 31a6678012..74a558e228 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -162,9 +162,12 @@ export async function finaliseRow( }) } - const response = await db.put(row) - // for response, calculate the formulas for the enriched row - enrichedRow._rev = response.rev + await db.put(row) + const retrieved = await db.tryGet(row._id) + enrichedRow = { + ...enrichedRow, + ...retrieved, + } enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, }) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index d9bdb8eb5b..14b524fd95 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -204,12 +204,11 @@ export async function inputProcessing( for (const [key, value] of Object.entries(clonedRow)) { const field = table.schema[key] + const isBuiltinColumn = isExternalTableID(table._id!) + ? isExternalColumnName(key) + : isInternalColumnName(key) // cleanse fields that aren't in the schema - if ( - !field && isExternalTableID(table._id!) - ? !isExternalColumnName(key) - : !isInternalColumnName(key) - ) { + if (!field && !isBuiltinColumn) { delete clonedRow[key] } // field isn't found - might be a built-in column, skip over it From a190feb4ea5053be8228f9bdc0bd8d83f1343890 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 26 Nov 2024 12:05:16 +0000 Subject: [PATCH 037/192] Fixing test cases. --- .../server/src/api/routes/tests/row.spec.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index aa78973ab4..c91be2b9ae 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2607,8 +2607,8 @@ if (descriptions.length) { name: "foo", description: "bar", tableId, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + createdAt: isInternal ? new Date().toISOString() : undefined, + updatedAt: isInternal ? new Date().toISOString() : undefined, }) }) @@ -2630,8 +2630,8 @@ if (descriptions.length) { id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + createdAt: isInternal ? new Date().toISOString() : undefined, + updatedAt: isInternal ? new Date().toISOString() : undefined, }) }) @@ -2654,8 +2654,8 @@ if (descriptions.length) { _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + createdAt: isInternal ? new Date().toISOString() : undefined, + updatedAt: isInternal ? new Date().toISOString() : undefined, }) }) @@ -2735,8 +2735,8 @@ if (descriptions.length) { id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + createdAt: isInternal ? new Date().toISOString() : undefined, + updatedAt: isInternal ? new Date().toISOString() : undefined, }) }) @@ -2761,8 +2761,8 @@ if (descriptions.length) { _rev: expect.any(String), id: isInternal ? undefined : expect.any(Number), type: isInternal ? "row" : undefined, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), + createdAt: isInternal ? new Date().toISOString() : undefined, + updatedAt: isInternal ? new Date().toISOString() : undefined, }) }) From a27c28ccdff8cf8b55b28d9c268642a33d571a30 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 26 Nov 2024 15:54:59 +0000 Subject: [PATCH 038/192] Fixing test case. --- packages/server/src/api/controllers/row/staticFormula.ts | 7 ++----- packages/server/src/api/routes/tests/row.spec.ts | 6 +++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 74a558e228..b9751b1ce2 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -15,7 +15,7 @@ import { } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" import isEqual from "lodash/isEqual" -import { cloneDeep } from "lodash/fp" +import { cloneDeep, merge } from "lodash/fp" import sdk from "../../../sdk" import * as pro from "@budibase/pro" @@ -164,10 +164,7 @@ export async function finaliseRow( await db.put(row) const retrieved = await db.tryGet(row._id) - enrichedRow = { - ...enrichedRow, - ...retrieved, - } + enrichedRow = merge(retrieved, enrichedRow) enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index c91be2b9ae..970e5f3619 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -85,7 +85,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) +const descriptions = datasourceDescribe({ only: [DatabaseName.SQS] }) if (descriptions.length) { describe.each(descriptions)( @@ -2571,12 +2571,14 @@ if (descriptions.length) { let tableId: string let o2mData: Row[] let m2mData: Row[] + let isRelationship: boolean beforeAll(async () => { const table = await config.api.table.save( defaultTable({ schema: relSchema() }) ) tableId = table._id! + isRelationship = relSchema().user.type === FieldType.LINK o2mData = [ await dataGenerator(o2mTable._id!), @@ -2763,6 +2765,8 @@ if (descriptions.length) { type: isInternal ? "row" : undefined, createdAt: isInternal ? new Date().toISOString() : undefined, updatedAt: isInternal ? new Date().toISOString() : undefined, + users: isRelationship ? undefined : [], + user: isRelationship ? undefined : [], }) }) From 594af57339a9cac886f766abc694475f18fc45e5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 16:28:22 +0000 Subject: [PATCH 039/192] wip --- packages/backend-core/src/sql/sql.ts | 60 +++-- packages/backend-core/src/sql/sqlTable.ts | 24 +- .../api/controllers/row/ExternalRequest.ts | 56 ++--- .../server/src/integrations/base/query.ts | 40 ++-- .../src/integrations/microsoftSqlServer.ts | 10 +- packages/server/src/integrations/mysql.ts | 8 +- packages/server/src/integrations/oracle.ts | 4 +- packages/server/src/integrations/postgres.ts | 4 +- .../basicFetchWithRelationships.json | 208 +++++++++-------- .../tests/sqlQueryJson/createSimple.json | 64 +++--- .../sqlQueryJson/createWithRelationships.json | 215 +++++++++-------- .../tests/sqlQueryJson/deleteSimple.json | 83 +++---- .../sqlQueryJson/enrichRelationship.json | 143 ++++++------ .../tests/sqlQueryJson/fetchManyToMany.json | 78 +++---- .../sqlQueryJson/filterByRelationship.json | 78 +++---- .../sqlQueryJson/manyRelationshipFilters.json | 168 +++++++------- .../sqlQueryJson/updateRelationship.json | 217 +++++++++--------- .../tests/sqlQueryJson/updateSimple.json | 217 +++++++++--------- .../src/sdk/app/rows/search/internal/sqs.ts | 23 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 28 ++- packages/server/src/sdk/app/rows/utils.ts | 36 ++- packages/types/src/sdk/datasources.ts | 4 +- packages/types/src/sdk/search.ts | 13 +- 23 files changed, 854 insertions(+), 927 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e97e31869c..c897c106b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -18,6 +18,7 @@ import { BasicOperator, BBReferenceFieldMetadata, CalculationType, + EnrichedQueryJson, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -27,7 +28,6 @@ import { LogicalOperator, Operation, prefixed, - QueryJson, QueryOptions, RangeOperator, RelationshipsJson, @@ -134,18 +134,18 @@ const allowEmptyRelationships: Record = { class InternalBuilder { private readonly client: SqlClient - private readonly query: QueryJson + private readonly query: EnrichedQueryJson private readonly splitter: dataFilters.ColumnSplitter private readonly knex: Knex - constructor(client: SqlClient, knex: Knex, query: QueryJson) { + constructor(client: SqlClient, knex: Knex, query: EnrichedQueryJson) { this.client = client this.query = query this.knex = knex this.splitter = new dataFilters.ColumnSplitter([this.table], { aliases: this.query.tableAliases, - columnPrefix: this.query.meta.columnPrefix, + columnPrefix: this.query.meta?.columnPrefix, }) } @@ -167,7 +167,7 @@ class InternalBuilder { } get table(): Table { - return this.query.meta.table + return this.query.table } get knexClient(): Knex.Client { @@ -273,8 +273,7 @@ class InternalBuilder { } private isFullSelectStatementRequired(): boolean { - const { meta } = this.query - for (let column of Object.values(meta.table.schema)) { + for (let column of Object.values(this.table.schema)) { if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { return true } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { @@ -285,14 +284,14 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource } = this.query + const { endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } const alias = this.getTableName(endpoint.entityId) - const schema = meta.table.schema + const schema = this.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw("??", [`${alias}.*`])] } @@ -1018,15 +1017,15 @@ class InternalBuilder { const name = tableOrName if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === name) { - table = this.query.meta.table - } else if (!this.query.meta.tables?.[name]) { + } else if (this.query.table.name === name) { + table = this.query.table + } else if (!this.query.tables[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return name } else { - table = this.query.meta.tables[name] + table = this.query.tables[name] } } else if (tableOrName) { table = tableOrName @@ -1238,7 +1237,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint, meta } = this.query + const { resource, tableAliases: aliases, endpoint, tables } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -1253,7 +1252,7 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } - const relatedTable = meta.tables?.[toTable] + const relatedTable = tables[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable @@ -1276,7 +1275,7 @@ class InternalBuilder { Math.floor(this.maxFunctionParameters() / 2) ) const fieldList = relationshipFields.map(field => - this.buildJsonField(relatedTable!, field) + this.buildJsonField(relatedTable, field) ) const fieldListFormatted = fieldList .map(f => { @@ -1472,9 +1471,7 @@ class InternalBuilder { if (this.client === SqlClient.ORACLE) { // Oracle doesn't seem to automatically insert nulls // if we don't specify them, so we need to do that here - for (const [column, schema] of Object.entries( - this.query.meta.table.schema - )) { + for (const [column, schema] of Object.entries(this.query.table.schema)) { if ( schema.constraints?.presence === true || schema.type === FieldType.FORMULA || @@ -1678,7 +1675,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes. * @return the query ready to be passed to the driver. */ - _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { + _query( + json: EnrichedQueryJson, + opts: QueryOptions = {} + ): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { client: this.getBaseSqlClient(), @@ -1728,34 +1728,30 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return this.convertToNative(query, opts) } - async getReturningRow(queryFn: QueryFunction, json: QueryJson) { + async getReturningRow(queryFn: QueryFunction, json: EnrichedQueryJson) { if (!json.extra || !json.extra.idFilter) { return {} } const input = this._query({ + ...json, endpoint: { ...json.endpoint, operation: Operation.READ, }, - resource: { - fields: [], - }, + resource: { fields: [] }, filters: json.extra?.idFilter, - paginate: { - limit: 1, - }, - meta: json.meta, + paginate: { limit: 1 }, }) return queryFn(input, Operation.READ) } // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row - checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta.table || !json.meta.table.primary) { + checkLookupKeys(id: any, json: EnrichedQueryJson) { + if (!id || !json.table.primary) { return json } - const primaryKey = json.meta.table.primary?.[0] + const primaryKey = json.table.primary[0] json.extra = { idFilter: { equal: { @@ -1768,7 +1764,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // this function recreates the returning functionality of postgres async queryWithReturning( - json: QueryJson, + json: EnrichedQueryJson, queryFn: QueryFunction, processFn: Function = (result: any) => result ) { diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 84f4e290aa..ee5dfbf453 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -3,13 +3,13 @@ import { FieldType, NumberFieldMetadata, Operation, - QueryJson, RelationshipType, RenameColumn, SqlQuery, Table, TableSourceType, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { breakExternalTableId, getNativeSql } from "./utils" import { helpers, utils } from "@budibase/shared-core" @@ -200,7 +200,7 @@ function buildUpdateTable( table: Table, tables: Record, oldTable: Table, - renamed: RenameColumn + renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable, renamed) @@ -238,11 +238,11 @@ class SqlTableQueryBuilder { * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. */ - _operation(json: QueryJson): Operation { + _operation(json: EnrichedQueryJson): Operation { return json.endpoint.operation } - _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] { + _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] { let client = knex({ client: this.sqlClient }).schema let schemaName = json?.endpoint?.schema if (schemaName) { @@ -250,7 +250,7 @@ class SqlTableQueryBuilder { } let query: Knex.SchemaBuilder - if (!json.table || !json.meta || !json.meta.tables) { + if (!json.table || !json.tables) { throw new Error("Cannot execute without table being specified") } if (json.table.sourceType === TableSourceType.INTERNAL) { @@ -259,14 +259,14 @@ class SqlTableQueryBuilder { switch (this._operation(json)) { case Operation.CREATE_TABLE: - query = buildCreateTable(client, json.table, json.meta.tables) + query = buildCreateTable(client, json.table, json.tables) break case Operation.UPDATE_TABLE: - if (!json.meta || !json.meta.table) { + if (!json.table) { throw new Error("Must specify old table for update") } // renameColumn does not work for MySQL, so return a raw query - if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) { const updatedColumn = json.meta.renamed.updated const tableName = schemaName ? `\`${schemaName}\`.\`${json.table.name}\`` @@ -280,14 +280,14 @@ class SqlTableQueryBuilder { query = buildUpdateTable( client, json.table, - json.meta.tables, - json.meta.table, - json.meta.renamed! + json.tables, + json.table, + json.meta?.renamed ) // renameColumn for SQL Server returns a parameterised `sp_rename` query, // which is not supported by SQL Server and gives a syntax error. - if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) { const oldColumn = json.meta.renamed.old const updatedColumn = json.meta.renamed.updated const tableName = schemaName diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 939ea399d9..baf1665cbe 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -42,7 +42,7 @@ import { sqlOutputProcessing, } from "./utils" import { - getDatasourceAndQuery, + enrichQueryJson, processRowCountResponse, } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" @@ -135,16 +135,9 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { return config } -function getEndpoint(tableId: string | undefined, operation: string) { - if (!tableId) { - throw new Error("Cannot get endpoint information - no table ID specified") - } +function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { - datasourceId: datasourceId, - entityId: tableName, - operation: operation as Operation, - } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -268,12 +261,9 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.DELETE), filters, - meta: { - table, - }, }) } else { return [] @@ -289,13 +279,10 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.UPDATE), body: { [colName]: null }, filters, - meta: { - table, - }, }) } else { return [] @@ -311,12 +298,9 @@ export class ExternalRequest { } async getRow(table: Table, rowId: string): Promise { - const response = await getDatasourceAndQuery({ + const response = await makeExternalQuery({ endpoint: getEndpoint(table._id!, Operation.READ), filters: this.prepareFilters(rowId, {}, table), - meta: { - table, - }, }) if (Array.isArray(response) && response.length > 0) { return response[0] @@ -490,16 +474,13 @@ export class ExternalRequest { if (!relatedTable) { throw new Error("unable to find related table") } - const response = await getDatasourceAndQuery({ - endpoint: endpoint, + const response = await makeExternalQuery({ + endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, - meta: { - table: relatedTable, - }, }) // this is the response from knex if no rows found const rows: Row[] = @@ -537,6 +518,11 @@ export class ExternalRequest { for (let relationship of relationships) { const { key, tableId, isUpdate, id, relationshipType, ...rest } = relationship + + if (!tableId) { + throw new Error("Table ID is unknown, cannot find table") + } + const body: { [key: string]: any } = processObjectSync(rest, row, {}) const linkTable = this.getTable(tableId) const relationshipPrimary = linkTable?.primary || [] @@ -583,14 +569,11 @@ export class ExternalRequest { const operation = isUpdate ? Operation.UPDATE : Operation.CREATE if (!existingRelationship) { promises.push( - getDatasourceAndQuery({ + makeExternalQuery({ endpoint: getEndpoint(tableId, operation), // if we're doing many relationships then we're writing, only one response body, filters: this.prepareFilters(id, {}, linkTable), - meta: { - table: linkTable, - }, }) ) } else { @@ -749,10 +732,6 @@ export class ExternalRequest { table ), }, - meta: { - table, - tables: this.tables, - }, } // remove any relationships that could block deletion @@ -773,8 +752,11 @@ export class ExternalRequest { response = [unprocessedRow] } else { response = env.SQL_ALIASING_DISABLE - ? await getDatasourceAndQuery(json) - : await aliasing.queryWithAliasing(json, makeExternalQuery) + ? await makeExternalQuery(json) + : await aliasing.queryWithAliasing( + await enrichQueryJson(json), + makeExternalQuery + ) } // if it's a counting operation there will be no more processing, just return the number diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 55886cd20f..2185eace2c 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -1,37 +1,27 @@ -import { - QueryJson, - Datasource, - DatasourcePlusQueryResponse, - RowOperations, -} from "@budibase/types" +import { DatasourcePlusQueryResponse, QueryJson } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" +import { enrichQueryJson } from "../../sdk/app/rows/utils" export async function makeExternalQuery( - datasource: Datasource, json: QueryJson ): Promise { - const entityId = json.endpoint.entityId, - tableName = json.meta.table.name, - tableId = json.meta.table._id - // case found during testing - make sure this doesn't happen again - if ( - RowOperations.includes(json.endpoint.operation) && - entityId !== tableId && - entityId !== tableName - ) { - throw new Error("Entity ID and table metadata do not align") - } - if (!datasource) { + const enrichedJson = await enrichQueryJson(json) + if (!enrichedJson.datasource) { throw new Error("No datasource provided for external query") } - datasource = await sdk.datasources.enrich(datasource) - const Integration = await getIntegration(datasource.source) + + enrichedJson.datasource = await sdk.datasources.enrich( + enrichedJson.datasource + ) + + const Integration = await getIntegration(enrichedJson.datasource.source) + // query is the opinionated function - if (Integration.prototype.query) { - const integration = new Integration(datasource.config) - return integration.query(json) - } else { + if (!Integration.prototype.query) { throw "Datasource does not support query." } + + const integration = new Integration(enrichedJson.datasource.config) + return integration.query(json) } diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 4997f3256d..a7a0b53312 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -4,9 +4,9 @@ import { DatasourceFieldType, DatasourcePlus, DatasourcePlusQueryResponse, + EnrichedQueryJson, Integration, Operation, - QueryJson, QueryType, Schema, SourceName, @@ -506,7 +506,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { return response.recordset || [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const schema = this.config.schema await this.connect() if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) { @@ -515,14 +515,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { const operation = this._operation(json) const queryFn = (query: any, op: string) => this.internalQuery(query, op) const processFn = (result: any) => { - if (json?.meta?.table && result.recordset) { + if (result.recordset) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result.recordset, json.tableAliases ) - } else if (result.recordset) { - return result.recordset } return [{ [operation]: true }] } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f78915fb49..9236409375 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -2,7 +2,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, TableSchema, @@ -15,6 +14,7 @@ import { DatasourcePlusQueryResponse, SqlQueryBinding, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -390,15 +390,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return results.length ? results : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { await this.connect() try { const queryFn = (query: any) => this.internalQuery(query, { connect: false, disableCoercion: true }) const processFn = (result: any) => { - if (json?.meta?.table && Array.isArray(result)) { + if (Array.isArray(result)) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result, json.tableAliases ) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index fcbc0731ea..b48a95e3e6 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,7 +3,6 @@ import { DatasourceFieldType, Integration, Operation, - QueryJson, QueryType, SqlQuery, Table, @@ -15,6 +14,7 @@ import { Row, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { buildExternalTableId, @@ -545,7 +545,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json) const input = this._query(json, { disableReturning: true }) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 28400f616f..164468b5ae 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -3,7 +3,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, DatasourcePlus, @@ -14,6 +13,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { return response.rows.length ? response.rows : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json).toLowerCase() const input = this._query(json) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index aa2cab3afd..27d6efec29 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -61,113 +61,111 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": ["personid"], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": ["support", "designer", "programmer", "qa"] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 33a88d30e1..474be32cf2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -5,10 +5,7 @@ "operation": "CREATE" }, "resource": { - "fields": [ - "a.name", - "a.age" - ] + "fields": ["a.name", "a.age"] }, "filters": {}, "relationships": [], @@ -24,41 +21,36 @@ } } }, - "meta": { - "table": { - "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", - "type": "table", - "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", - "sourceType": "external", - "primary": [ - "name", - "age" - ], - "name": "people", - "schema": { - "name": { - "type": "string", - "externalType": "varchar", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": true - } - }, - "age": { - "type": "number", - "externalType": "int", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } + "table": { + "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", + "type": "table", + "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "sourceType": "external", + "primary": ["name", "age"], + "name": "people", + "schema": { + "name": { + "type": "string", + "externalType": "varchar", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": true } }, - "primaryDisplay": "name" - } + "age": { + "type": "number", + "externalType": "int", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + } + }, + "primaryDisplay": "name" }, "tableAliases": { "people": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 82d85c417b..fbe0b468e9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -52,122 +52,113 @@ "idFilter": {} }, "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index d6e099c4b6..d9fc34165f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -5,11 +5,7 @@ "operation": "DELETE" }, "resource": { - "fields": [ - "a.keyparttwo", - "a.keypartone", - "a.name" - ] + "fields": ["a.keyparttwo", "a.keypartone", "a.name"] }, "filters": { "equal": { @@ -26,50 +22,45 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", - "primary": [ - "keypartone", - "keyparttwo" - ], - "name": "compositetable", - "schema": { - "keyparttwo": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keyparttwo", - "constraints": { - "presence": true - } - }, - "keypartone": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keypartone", - "constraints": { - "presence": true - } - }, - "name": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": false - } + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", + "primary": ["keypartone", "keyparttwo"], + "name": "compositetable", + "schema": { + "keyparttwo": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keyparttwo", + "constraints": { + "presence": true } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "keypartone" - } + "keypartone": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keypartone", + "constraints": { + "presence": true + } + }, + "name": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": false + } + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "keypartone" }, "tableAliases": { "compositetable": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index d71f0552c6..f24b46859f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -17,10 +17,7 @@ }, "filters": { "oneOf": { - "taskid": [ - 1, - 2 - ] + "taskid": [1, 2] } }, "relationships": [ @@ -42,82 +39,78 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": [ - "taskid" - ], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "executorid", - "constraints": { - "presence": false - } - }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "executorid", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false + } + }, + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", "products": "b", "products_tasks": "c" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index d81c8cf6d0..b746e05c5e 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -56,48 +56,46 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "productname" }, "tableAliases": { "products": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index 36010bf61d..6a9717f5dd 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -46,47 +46,45 @@ "tasks": "b", "products": "a" }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "productname" } } diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index 01b85f433f..23f56796a9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -102,94 +102,92 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": ["taskid"], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "name": "executorid", - "constraints": { - "presence": false - }, - "autocolumn": true, - "autoReason": "foreign_key" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "name": "executorid", + "constraints": { + "presence": false }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" - }, - "tasksToExecute": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToExecute", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c0f440590bda04f28846242156c1dd60b", - "foreignKey": "executorid", - "fieldName": "personid" - }, - "tasksToQA": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToQA", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c5fdf453a0ba743d58e29491d174c974b", - "foreignKey": "qaid", - "fieldName": "personid" + "autocolumn": true, + "autoReason": "foreign_key" + }, + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + }, + "tasksToExecute": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToExecute", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c0f440590bda04f28846242156c1dd60b", + "foreignKey": "executorid", + "fieldName": "personid" + }, + "tasksToQA": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToQA", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c5fdf453a0ba743d58e29491d174c974b", + "foreignKey": "qaid", + "fieldName": "personid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a160bfbb3e..a12b842afc 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -1,8 +1,8 @@ import { Aggregation, CalculationType, - Datasource, DocumentType, + EnrichedQueryJson, FieldType, isLogicalSearchOperator, Operation, @@ -38,7 +38,7 @@ import { generateJunctionTableID } from "../../../../../db/utils" import AliasTables from "../../sqlAlias" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" -import { processRowCountResponse } from "../../utils" +import { enrichQueryJson, processRowCountResponse } from "../../utils" import { dataFilters, helpers, @@ -223,18 +223,18 @@ function reverseUserColumnMapping(rows: Row[], table?: Table) { } function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[] ): Promise function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts: { countTotalRows: true } ): Promise async function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts?: { countTotalRows?: boolean } @@ -246,7 +246,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT } - const processSQLQuery = async (_: Datasource, json: QueryJson) => { + const processSQLQuery = async (json: EnrichedQueryJson) => { const query = builder._query(json, { disableReturning: true, }) @@ -281,7 +281,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { return processRowCountResponse(response) } else if (Array.isArray(response)) { - return reverseUserColumnMapping(response, json.meta.table) + return reverseUserColumnMapping(response, json.table) } return response } @@ -372,10 +372,7 @@ export async function search( operation: Operation.READ, }, filters: searchFilters, - table, meta: { - table, - tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, resource: { @@ -427,11 +424,13 @@ export async function search( } } + const enrichedRequest = await enrichQueryJson(request) + try { const [rows, totalRows] = await Promise.all([ - runSqlQuery(request, allTables, relationships), + runSqlQuery(enrichedRequest, allTables, relationships), options.countRows - ? runSqlQuery(request, allTables, relationships, { + ? runSqlQuery(enrichedRequest, allTables, relationships, { countTotalRows: true, }) : Promise.resolve(undefined), diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 535709791d..c748940add 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -1,11 +1,13 @@ import { Datasource, DatasourcePlusQueryResponse, + EnrichedQueryJson, Operation, QueryJson, Row, SearchFilters, SqlClient, + Table, } from "@budibase/types" import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" @@ -15,7 +17,6 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - datasource: Datasource, json: QueryJson ) => Promise @@ -99,7 +100,11 @@ export default class AliasTables { return true } - getAlias(tableName: string) { + getAlias(tableName: string | Table) { + if (typeof tableName === "object") { + tableName = tableName.name + } + if (this.aliases[tableName]) { return this.aliases[tableName] } @@ -177,7 +182,7 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson, + json: EnrichedQueryJson, queryFn: PerformQueryFunction ): Promise { const datasourceId = json.endpoint.datasourceId @@ -215,14 +220,7 @@ export default class AliasTables { } json.filters = aliasFilters(json.filters) } - if (json.meta?.table) { - this.getAlias(json.meta.table.name) - } - if (json.meta?.tables) { - Object.keys(json.meta.tables).forEach(tableName => - this.getAlias(tableName) - ) - } + if (json.relationships) { json.relationships = json.relationships.map(relationship => ({ ...relationship, @@ -233,6 +231,12 @@ export default class AliasTables { ]), })) } + + this.getAlias(json.table) + for (const tableName of Object.keys(json.tables)) { + this.getAlias(tableName) + } + // invert and return const invertedTableAliases: Record = {} for (let [key, value] of Object.entries(this.tableAliases)) { @@ -241,7 +245,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse = await queryFn(datasource, json) + let response = await queryFn(json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bded6a7a18..f7646a3a85 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,13 +14,13 @@ import { SqlClient, ArrayOperator, ViewV2, + EnrichedQueryJson, } from "@budibase/types" -import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" -import { docIds, sql } from "@budibase/backend-core" +import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" @@ -73,18 +73,32 @@ export function processRowCountResponse( } } -export async function getDatasourceAndQuery( +export async function enrichQueryJson( json: QueryJson -): Promise { - const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) - const table = datasource.entities?.[json.endpoint.entityId] - if (!json.meta && table) { - json.meta = { - table, +): Promise { + let datasource: Datasource | undefined = undefined + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId) + } + + let tables: Record + if (datasource) { + tables = datasource.entities || {} + } else { + tables = {} + for (const table of await sdk.tables.getAllInternalTables()) { + tables[table._id!] = table } } - return makeExternalQuery(datasource, json) + + const table = tables[json.endpoint.entityId] + + return { + table, + tables, + datasource, + ...json, + } } export function cleanExportRows( diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index bdad9c4825..8b0574bd7c 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,5 +1,5 @@ import { Table, Row } from "../documents" -import { QueryJson } from "./search" +import { EnrichedQueryJson } from "./search" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -207,7 +207,7 @@ export interface DatasourcePlus extends IntegrationBase { // this returns the format of the identifier getBindingIdentifier(): string getStringConcat(parts: string[]): string - query(json: QueryJson): Promise + query(json: EnrichedQueryJson): Promise buildSchema( datasourceId: string, entities: Record diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d6dcb0a2fa..d87472cca9 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -1,5 +1,5 @@ import { Operation } from "./datasources" -import { Row, Table, DocumentType } from "../documents" +import { Row, DocumentType, Table, Datasource } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" import { Aggregation } from "./row" @@ -171,10 +171,7 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: Row | Row[] - table?: Table - meta: { - table: Table - tables?: Record + meta?: { renamed?: RenameColumn // can specify something that columns could be prefixed with columnPrefix?: string @@ -186,6 +183,12 @@ export interface QueryJson { tableAliases?: Record } +export interface EnrichedQueryJson extends QueryJson { + table: Table + tables: Record + datasource?: Datasource +} + export interface QueryOptions { disableReturning?: boolean disableBindings?: boolean From e4ae16cc3954875a0b127398108f48b004020102 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 16:28:55 +0000 Subject: [PATCH 040/192] wip --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 25dd40ee12..749b3861b5 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd +Subproject commit 749b3861b579b391496bd8cbce1c473e2f978cc9 From c35e443225d519202d6b65c01f25a0d3b6e7940d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:05:47 +0000 Subject: [PATCH 041/192] Making progress toward type checking successfully. --- packages/backend-core/src/sql/sql.ts | 3 + .../server/src/api/controllers/datasource.ts | 6 +- .../api/controllers/table/ExternalRequest.ts | 17 +- .../server/src/integrations/googlesheets.ts | 6 +- .../server/src/integrations/tests/sql.spec.ts | 268 ------------------ packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- 6 files changed, 16 insertions(+), 286 deletions(-) delete mode 100644 packages/server/src/integrations/tests/sql.spec.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c897c106b0..5333bfb591 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1253,6 +1253,9 @@ class InternalBuilder { continue } const relatedTable = tables[toTable] + if (!relatedTable) { + throw new Error(`related table "${toTable}" not found in datasource`) + } const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 794e2dfddd..9f4fff4c46 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,11 +23,13 @@ import { Table, RowValue, DynamicVariable, + QueryJson, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" +import { makeExternalQuery } from "src/integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() @@ -297,10 +299,10 @@ export async function find(ctx: UserCtx) { } // dynamic query functionality -export async function query(ctx: UserCtx) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { - ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) + ctx.body = await makeExternalQuery(queryJson) } catch (err: any) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 1e57ea3294..dc32c79c5a 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,8 +11,6 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, - tables: Record, - oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { @@ -21,17 +19,12 @@ export async function makeTableRequest( entityId: table._id!, operation, }, - meta: { - table, - tables, - }, - table, - } - if (oldTable) { - json.meta!.table = oldTable } if (renamed) { - json.meta!.renamed = renamed + if (!json.meta) { + json.meta = {} + } + json.meta.renamed = renamed } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json) } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 5f61791683..d8bf0e376d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -7,7 +7,6 @@ import { Integration, Operation, PaginationJson, - QueryJson, QueryType, Row, Schema, @@ -18,6 +17,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, BBReferenceFieldSubType, + EnrichedQueryJson, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { @@ -381,7 +381,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return { tables: externalTables, errors } } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const sheet = json.endpoint.entityId switch (json.endpoint.operation) { case Operation.CREATE: @@ -400,7 +400,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rowIndex: json.extra?.idFilter?.equal?.rowNumber, sheet, row: json.body, - table: json.meta.table, + table: json.table, }) case Operation.DELETE: return this.delete({ diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts deleted file mode 100644 index 2b3649161c..0000000000 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ /dev/null @@ -1,268 +0,0 @@ -import { - FieldType, - Operation, - PaginationJson, - QueryJson, - SearchFilters, - SortJson, - SqlClient, - Table, - TableSourceType, -} from "@budibase/types" -import { sql } from "@budibase/backend-core" -import { merge } from "lodash" - -const Sql = sql.Sql - -const TABLE_NAME = "test" -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: { - id: { - name: "id", - type: FieldType.NUMBER, - }, - }, - name: TABLE_NAME, - primary: ["id"], -} - -const ORACLE_TABLE: Partial = { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, -} - -function endpoint(table: string, operation: Operation) { - return { - datasourceId: "Postgres", - operation: operation, - entityId: table || TABLE_NAME, - } -} - -function generateReadJson({ - table, - fields, - filters, - sort, - paginate, -}: { - table?: Partial
- fields?: string[] - filters?: SearchFilters - sort?: SortJson - paginate?: PaginationJson -} = {}): QueryJson { - let tableObj: Table = { ...TABLE } - if (table) { - tableObj = merge(TABLE, table) - } - return { - endpoint: endpoint(tableObj.name || TABLE_NAME, Operation.READ), - resource: { - fields: fields || [], - }, - filters: filters || {}, - sort: sort || {}, - paginate: paginate || undefined, - meta: { - table: tableObj, - }, - } -} - -function generateRelationshipJson(config: { schema?: string } = {}): QueryJson { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "brands", - operation: Operation.READ, - schema: config.schema, - }, - resource: { - fields: [ - "brands.brand_id", - "brands.brand_name", - "products.product_id", - "products.product_name", - "products.brand_id", - ], - }, - filters: {}, - sort: {}, - relationships: [ - { - from: "brand_id", - to: "brand_id", - tableName: "products", - column: "products", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -function generateManyRelationshipJson(config: { schema?: string } = {}) { - return { - endpoint: { - datasourceId: "Postgres", - entityId: "stores", - operation: "READ", - schema: config.schema, - }, - resource: { - fields: [ - "stores.store_id", - "stores.store_name", - "products.product_id", - "products.product_name", - ], - }, - filters: {}, - sort: {}, - paginate: {}, - relationships: [ - { - from: "store_id", - to: "product_id", - tableName: "products", - column: "products", - through: "stocks", - fromPrimary: "store_id", - toPrimary: "product_id", - }, - ], - extra: { idFilter: {} }, - meta: { - table: TABLE, - }, - } -} - -describe("SQL query builder", () => { - const relationshipLimit = 500 - const limit = 500 - const client = SqlClient.POSTGRES - let sql: any - - beforeEach(() => { - sql = new Sql(client, limit) - }) - - it("should add the schema to the LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson({ schema: "production" })) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should handle if the schema is not present when doing a LEFT JOIN", () => { - const query = sql._query(generateRelationshipJson()) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, - }) - }) - - it("should add the schema to both the toTable and throughTable in many-to-many join", () => { - const query = sql._query( - generateManyRelationshipJson({ schema: "production" }) - ) - expect(query).toEqual({ - bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "stores".* from "production"."stores" order by "test"."id" asc limit $1) select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $2) as "products") as "products" from "paginated" as "stores" order by "test"."id" asc`, - }) - }) - - it("should lowercase the values for Oracle LIKE statements", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - string: { - name: "John", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - contains: { - age: [20, 25], - name: ["John", "Mary"], - }, - }, - }) - ) - const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] - expect(query).toEqual({ - bindings: [...filterSet, limit], - sql: `select * from (select * from "test" where ((COALESCE(LOWER("test"."age"), '') like :1 and COALESCE(LOWER("test"."age"), '') like :2)) and ((COALESCE(LOWER("test"."name"), '') like :3 and COALESCE(LOWER("test"."name"), '') like :4)) order by "test"."id" asc) where rownum <= :5`, - }) - - query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - filters: { - fuzzy: { - name: "Jo", - }, - }, - }) - ) - expect(query).toEqual({ - bindings: [`%jo%`, limit], - sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - equal: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, - }) - }) - - it("should use an oracle compatible coalesce query for oracle when using the not equals filter", () => { - let query = new Sql(SqlClient.ORACLE, limit)._query( - generateReadJson({ - table: ORACLE_TABLE, - filters: { - notEqual: { - name: "John", - }, - }, - }) - ) - - expect(query).toEqual({ - bindings: ["John", limit], - sql: `select * from (select * from "test" where (to_char("test"."name") is not null and to_char("test"."name") != :1) or to_char("test"."name") is null order by "test"."id" asc) where rownum <= :2`, - }) - }) -}) diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index c748940add..c14307360b 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -17,7 +17,7 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - json: QueryJson + json: EnrichedQueryJson ) => Promise const WRITE_OPERATIONS: Operation[] = [ From 8e756ccf848943a07526f929fc288ab9775507bb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:09:32 +0000 Subject: [PATCH 042/192] Type checks pass. --- .../server/src/integrations/tests/sqlAlias.spec.ts | 8 +++----- .../server/src/sdk/app/tables/external/index.ts | 13 +++---------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 739d3a4aee..66340bc794 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -7,6 +7,7 @@ import { Table, TableSourceType, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { sql } from "@budibase/backend-core" import { join } from "path" @@ -35,8 +36,8 @@ describe("Captures of real examples", () => { const relationshipLimit = 500 const primaryLimit = 100 - function getJson(name: string): QueryJson { - return require(join(__dirname, "sqlQueryJson", name)) as QueryJson + function getJson(name: string) { + return require(join(__dirname, "sqlQueryJson", name)) as EnrichedQueryJson } describe("create", () => { @@ -252,9 +253,6 @@ describe("Captures of real examples", () => { resource: { fields, }, - meta: { - table: TABLE, - }, } } diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 941d193b94..5d8f03a7a6 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -241,19 +241,12 @@ export async function save( } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest( - datasource, - operation, - tableToSave, - tables, - oldTable, - opts?.renaming - ) + await makeTableRequest(datasource, operation, tableToSave, opts?.renaming) // update any extra tables (like foreign keys in other tables) for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable) + await makeTableRequest(datasource, op, extraTable) } // make sure the constrained list, all still exist @@ -292,7 +285,7 @@ export async function destroy(datasourceId: string, table: Table) { const operation = Operation.DELETE_TABLE if (tables) { - await makeTableRequest(datasource, operation, table, tables) + await makeTableRequest(datasource, operation, table) cleanupRelationships(table, tables, { deleting: true }) delete tables[table.name] datasource.entities = tables From f21199011788c2f6e69e77d5ea983f3f8662961b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 17:36:17 +0000 Subject: [PATCH 043/192] Some sqs fixes. --- packages/backend-core/src/sql/sql.ts | 13 ++++++++++--- packages/server/src/api/controllers/datasource.ts | 2 +- packages/server/src/api/routes/tests/row.spec.ts | 2 +- .../server/src/sdk/app/rows/search/internal/sqs.ts | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5333bfb591..baeeefbb81 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,6 +3,7 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, + isInternalTableID, isInvalidISODateString, isValidFilter, isValidISODateString, @@ -1192,7 +1193,7 @@ class InternalBuilder { private buildJsonField(table: Table, field: string): [string, Knex.Raw] { const parts = field.split(".") - const baseName = parts[parts.length - 1] + let baseName = parts[parts.length - 1] let unaliased: string let tableField: string @@ -1205,10 +1206,16 @@ class InternalBuilder { tableField = unaliased } - const schema = table.schema[baseName] + if (this.query.meta?.columnPrefix) { + baseName = baseName.replace(this.query.meta.columnPrefix, "") + } let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { + // Internal tables have special _id, _rev, createdAt, and updatedAt fields + // that do not appear in the schema, meaning schema could actually be + // undefined. + const schema: FieldSchema | undefined = table.schema[baseName] + if (schema && schema.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } return [unaliased, identifier] diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 9f4fff4c46..01d81425b6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -29,7 +29,7 @@ import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" -import { makeExternalQuery } from "src/integrations/base/query" +import { makeExternalQuery } from "../../integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9e84e74d05..1ab12feebb 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3333,7 +3333,7 @@ if (descriptions.length) { }) isInternal && - it("should coerce a static handlebars formula", async () => { + it.only("should coerce a static handlebars formula", async () => { await updateFormulaColumn(encodeJS("return 1"), { responseType: FieldType.NUMBER, formulaType: FormulaType.STATIC, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a12b842afc..aec934ae4e 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -494,6 +494,7 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) + // throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) + throw err } } From 92e92c8b7f5e012993b8c7dd875585caed305d0f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 26 Nov 2024 18:15:08 +0000 Subject: [PATCH 044/192] Updating sql alias test case. --- packages/backend-core/src/sql/sql.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 59 +++++++++++++------ .../sqlQueryJson/createWithRelationships.json | 2 +- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c897c106b0..f6acfbffe2 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1208,7 +1208,7 @@ class InternalBuilder { const schema = table.schema[baseName] let identifier = this.rawQuotedIdentifier(tableField) - if (schema.type === FieldType.BIGINT) { + if (schema?.type === FieldType.BIGINT) { identifier = this.castIntToString(identifier) } return [unaliased, identifier] diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 739d3a4aee..951c24a65f 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -1,12 +1,13 @@ import { Datasource, Operation, - QueryJson, SourceName, SqlQuery, Table, TableSourceType, SqlClient, + EnrichedQueryJson, + TableSchema, } from "@budibase/types" import { sql } from "@budibase/backend-core" import { join } from "path" @@ -16,17 +17,21 @@ import sdk from "../../sdk" const Sql = sql.Sql // this doesn't exist strictly -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: {}, - name: "tableName", - primary: ["id"], -} +const TABLE = buildTable("tableName", {}) const AliasTables = sdk.rows.AliasTables +function buildTable(name: string, schema: TableSchema): Table { + return { + type: "table", + sourceType: TableSourceType.EXTERNAL, + sourceId: "SOURCE_ID", + schema: schema, + name: name, + primary: ["id"], + } +} + function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") } @@ -35,8 +40,22 @@ describe("Captures of real examples", () => { const relationshipLimit = 500 const primaryLimit = 100 - function getJson(name: string): QueryJson { - return require(join(__dirname, "sqlQueryJson", name)) as QueryJson + function getJson(name: string): EnrichedQueryJson { + // tables aren't fully specified in the test JSON + const base = require(join(__dirname, "sqlQueryJson", name)) as Omit< + EnrichedQueryJson, + "tables" + > + const tables: Record = { [base.table.name]: base.table } + if (base.relationships) { + for (let { tableName } of base.relationships) { + tables[tableName] = buildTable(tableName, {}) + } + } + return { + ...base, + tables: tables, + } } describe("create", () => { @@ -63,7 +82,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` + `select json_agg(json_build_object('executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'completed',"b"."completed",'completed',"b"."completed",'taskname',"b"."taskname",'taskname',"b"."taskname"` ) ), }) @@ -94,7 +113,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) + select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -212,8 +231,8 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: multiline( - `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 - and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` + `select top (@p0) * from [people] as [a] where CASE WHEN [a].[name] = @p1 THEN 1 ELSE 0 END = 1 and + CASE WHEN [a].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [a].[name] asc` ), bindings: [1, "Test", 22], }) @@ -246,15 +265,17 @@ describe("Captures of real examples", () => { } } - function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { + function getQuery( + op: Operation, + fields: string[] = ["a"] + ): EnrichedQueryJson { return { endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, }, - meta: { - table: TABLE, - }, + table: TABLE, + tables: { [TABLE.name]: TABLE }, } } diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index fbe0b468e9..060197dd01 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -51,7 +51,7 @@ "extra": { "idFilter": {} }, - "meta": { + "table": { "type": "table", "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", "primary": ["personid"], From bd17770e4ec3cc9414ca97e4ce3de3dc923eb9b4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 12:21:27 +0000 Subject: [PATCH 045/192] Fix _some_ sqs problems. --- .../server/src/api/routes/tests/row.spec.ts | 6 +++--- .../src/sdk/app/rows/search/internal/sqs.ts | 19 +++++-------------- packages/server/src/sdk/app/rows/utils.ts | 16 ++++++++++++---- packages/server/src/sdk/app/tables/getters.ts | 1 + 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 1ab12feebb..6756d2e462 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3268,7 +3268,7 @@ if (descriptions.length) { formula: { name: "formula", type: FieldType.FORMULA, - formula: formula, + formula, responseType: opts?.responseType, formulaType: opts?.formulaType || FormulaType.DYNAMIC, }, @@ -3276,7 +3276,7 @@ if (descriptions.length) { }) } - it("should be able to search for rows containing formulas", async () => { + it.only("should be able to search for rows containing formulas", async () => { const { rows } = await config.api.row.search(table._id!) expect(rows.length).toBe(1) expect(rows[0].links.length).toBe(1) @@ -3333,7 +3333,7 @@ if (descriptions.length) { }) isInternal && - it.only("should coerce a static handlebars formula", async () => { + it("should coerce a static handlebars formula", async () => { await updateFormulaColumn(encodeJS("return 1"), { responseType: FieldType.NUMBER, formulaType: FormulaType.STATIC, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index aec934ae4e..1a55fdf99c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -180,19 +180,6 @@ function cleanupFilters(filters: SearchFilters, allTables: Table[]) { return filters } -function buildTableMap(tables: Table[]) { - const tableMap: Record = {} - for (let table of tables) { - // update the table name, should never query by name for SQLite - table.originalName = table.name - table.name = table._id! - // need a primary for sorting, lookups etc - table.primary = ["_id"] - tableMap[table._id!] = table - } - return tableMap -} - // table is only needed to handle relationships function reverseUserColumnMapping(rows: Row[], table?: Table) { const prefixLength = USER_COLUMN_PREFIX.length @@ -315,7 +302,11 @@ export async function search( } const allTables = await sdk.tables.getAllInternalTables() - const allTablesMap = buildTableMap(allTables) + const allTablesMap = allTables.reduce((acc, table) => { + acc[table._id!] = table + return acc + }, {} as Record) + // make sure we have the mapped/latest table if (table._id) { table = allTablesMap[table._id] diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index f7646a3a85..eb6859c58b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -73,6 +73,17 @@ export function processRowCountResponse( } } +function processInternalTables(tables: Table[]) { + const tableMap: Record = {} + for (let table of tables) { + // update the table name, should never query by name for SQLite + table.originalName = table.name + table.name = table._id! + tableMap[table._id!] = table + } + return tableMap +} + export async function enrichQueryJson( json: QueryJson ): Promise { @@ -85,10 +96,7 @@ export async function enrichQueryJson( if (datasource) { tables = datasource.entities || {} } else { - tables = {} - for (const table of await sdk.tables.getAllInternalTables()) { - tables[table._id!] = table - } + tables = processInternalTables(await sdk.tables.getAllInternalTables()) } const table = tables[json.endpoint.entityId] diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 1ad82b8e42..79091ea660 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -46,6 +46,7 @@ export async function processTable(table: Table): Promise
{ const processed: Table = { ...table, type: "table", + primary: ["_id"], // internal tables must always use _id as primary key sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, sql: true, From 504ddba9b98599e973f35ca73e47cae271eadd9c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 12:40:34 +0000 Subject: [PATCH 046/192] SQS row tests passing. --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 6756d2e462..b371ab1f8a 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3276,7 +3276,7 @@ if (descriptions.length) { }) } - it.only("should be able to search for rows containing formulas", async () => { + it("should be able to search for rows containing formulas", async () => { const { rows } = await config.api.row.search(table._id!) expect(rows.length).toBe(1) expect(rows[0].links.length).toBe(1) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 1a55fdf99c..ba5614edca 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -228,7 +228,7 @@ async function runSqlQuery( ) { const relationshipJunctionTableIds = relationships.map(rel => rel.through!) const alias = new AliasTables( - tables.map(table => table.name).concat(relationshipJunctionTableIds) + tables.map(table => table._id!).concat(relationshipJunctionTableIds) ) if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT From 332505388dba3b2a35de2e32c120d04ead4ee427 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 13:14:42 +0000 Subject: [PATCH 047/192] Fix postgres tests. --- .../server/src/api/controllers/datasource.ts | 4 ++-- .../src/api/controllers/row/ExternalRequest.ts | 4 ++-- .../api/controllers/table/ExternalRequest.ts | 2 +- .../routes/tests/queries/generic-sql.spec.ts | 2 +- packages/server/src/integrations/base/query.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 15 +-------------- .../src/sdk/app/rows/search/internal/sqs.ts | 2 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 11 +++-------- packages/server/src/sdk/app/rows/utils.ts | 18 +++++++++++++++--- .../src/tests/utilities/api/datasource.ts | 7 ++----- packages/types/src/sdk/search.ts | 6 +++++- 11 files changed, 34 insertions(+), 39 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 01d81425b6..82eadca312 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,7 +23,7 @@ import { Table, RowValue, DynamicVariable, - QueryJson, + QueryJsonRequest, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" @@ -299,7 +299,7 @@ export async function find(ctx: UserCtx) { } // dynamic query functionality -export async function query(ctx: UserCtx) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { ctx.body = await makeExternalQuery(queryJson) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index baf1665cbe..422896cd52 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -137,7 +137,7 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { datasourceId, entityId: tableName, operation } + return { datasource: datasourceId, entityId: tableName, operation } } function isOneSide( @@ -706,7 +706,7 @@ export class ExternalRequest { let json: QueryJson = { endpoint: { - datasourceId: this.datasource._id!, + datasource: this.datasource, entityId: table.name, operation, }, diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index dc32c79c5a..ca4fcb1d16 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -15,7 +15,7 @@ export async function makeTableRequest( ) { const json: QueryJson = { endpoint: { - datasourceId: datasource._id!, + datasource, entityId: table._id!, operation, }, diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 44b21e0350..9130034501 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -837,7 +837,7 @@ if (descriptions.length) { const res = await config.api.datasource.query({ endpoint: { - datasourceId: datasource._id!, + datasource: datasource._id!, operation: Operation.READ, entityId, }, diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 2185eace2c..6555a5de51 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -23,5 +23,5 @@ export async function makeExternalQuery( } const integration = new Integration(enrichedJson.datasource.config) - return integration.query(json) + return integration.query(enrichedJson) } diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 66340bc794..73a9c9fe8d 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -4,8 +4,6 @@ import { QueryJson, SourceName, SqlQuery, - Table, - TableSourceType, SqlClient, EnrichedQueryJson, } from "@budibase/types" @@ -15,17 +13,6 @@ import { generator } from "@budibase/backend-core/tests" import sdk from "../../sdk" const Sql = sql.Sql - -// this doesn't exist strictly -const TABLE: Table = { - type: "table", - sourceType: TableSourceType.EXTERNAL, - sourceId: "SOURCE_ID", - schema: {}, - name: "tableName", - primary: ["id"], -} - const AliasTables = sdk.rows.AliasTables function multiline(sql: string) { @@ -249,7 +236,7 @@ describe("Captures of real examples", () => { function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson { return { - endpoint: { datasourceId: "", entityId: "", operation: op }, + endpoint: { datasource: "", entityId: "", operation: op }, resource: { fields, }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index ba5614edca..d6ac2a5862 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -358,7 +358,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasourceId: SQS_DATASOURCE_INTERNAL, + datasource: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index c14307360b..2f9bea3c91 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -9,11 +9,8 @@ import { SqlClient, Table, } from "@budibase/types" -import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" -import datasources from "../datasources" -import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( @@ -185,14 +182,12 @@ export default class AliasTables { json: EnrichedQueryJson, queryFn: PerformQueryFunction ): Promise { - const datasourceId = json.endpoint.datasourceId - const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL - let aliasingEnabled: boolean, datasource: Datasource + const datasource = json.datasource + const isSqs = datasource === undefined + let aliasingEnabled: boolean if (isSqs) { aliasingEnabled = this.isAliasingEnabled(json) - datasource = BudibaseInternalDB } else { - datasource = await datasources.get(datasourceId) aliasingEnabled = this.isAliasingEnabled(json, datasource) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index eb6859c58b..17bbebb251 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -23,6 +23,8 @@ import { isSQL } from "../../../integrations/utils" import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" +import { breakExternalTableId } from "@budibase/backend-core/src/sql/utils" +import { isDatasourceId } from "@budibase/backend-core/src/docIds" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -88,8 +90,18 @@ export async function enrichQueryJson( json: QueryJson ): Promise { let datasource: Datasource | undefined = undefined - if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { - datasource = await sdk.datasources.get(json.endpoint.datasourceId) + let entityId = json.endpoint.entityId + if (typeof json.endpoint.datasource === "string") { + if (json.endpoint.datasource !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasource, { + enriched: true, + }) + } + } else { + datasource = json.endpoint.datasource + if (isDatasourceId(entityId)) { + entityId = breakExternalTableId(entityId).tableName + } } let tables: Record @@ -99,7 +111,7 @@ export async function enrichQueryJson( tables = processInternalTables(await sdk.tables.getAllInternalTables()) } - const table = tables[json.endpoint.entityId] + const table = tables[entityId] return { table, diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 87f03c8a6f..f4adaefa24 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -4,7 +4,7 @@ import { Datasource, FetchDatasourceInfoResponse, FieldType, - QueryJson, + QueryJsonRequest, RelationshipType, UpdateDatasourceRequest, UpdateDatasourceResponse, @@ -69,10 +69,7 @@ export class DatasourceAPI extends TestAPI { return await this._get(`/api/datasources`, { expectations }) } - query = async ( - query: Omit & Partial>, - expectations?: Expectations - ) => { + query = async (query: QueryJsonRequest, expectations?: Expectations) => { return await this._post(`/api/datasources/query`, { body: query, expectations, diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d87472cca9..72f7de455a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -158,7 +158,7 @@ export interface ManyToManyRelationshipJson { export interface QueryJson { endpoint: { - datasourceId: string + datasource: string | Datasource entityId: string operation: Operation schema?: string @@ -189,6 +189,10 @@ export interface EnrichedQueryJson extends QueryJson { datasource?: Datasource } +export interface QueryJsonRequest extends Omit { + endpoint: QueryJson["endpoint"] & { datasource: string } +} + export interface QueryOptions { disableReturning?: boolean disableBindings?: boolean From 525cf2ab09085c48e189d04cb3653998da2ee442 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 13:16:21 +0000 Subject: [PATCH 048/192] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 749b3861b5..06e4506cef 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 749b3861b579b391496bd8cbce1c473e2f978cc9 +Subproject commit 06e4506cef37110e509a0ef0b9811ac07f8b2844 From 49af5501f1bbd09009238715a355cd5a025365eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 15:05:37 +0000 Subject: [PATCH 049/192] Adding test container debug. --- .github/workflows/budibase_ci.yml | 3 ++- yarn.lock | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1258bddcca..7c48bbc774 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -26,7 +26,8 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }} - IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} + IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') } + DEBUG: testcontainers* jobs: lint: diff --git a/yarn.lock b/yarn.lock index dd01832bba..edd5550b30 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12635,6 +12635,11 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== +husky@^9.1.4: + version "9.1.7" + resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" + integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== + ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" From f556a2b3ce373423615e745711467569051cfd30 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 15:11:58 +0000 Subject: [PATCH 050/192] Fix. --- .github/workflows/budibase_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 7c48bbc774..05837f217c 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -26,7 +26,7 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }} - IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') } + IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} DEBUG: testcontainers* jobs: From 70cdb99583c4b7ee0d72f2cec47c02181c2e9651 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 15:30:48 +0000 Subject: [PATCH 051/192] Increasing timeouts. --- packages/server/src/integrations/tests/utils/index.ts | 3 +++ packages/server/src/integrations/tests/utils/oracle.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index dcdaece191..e5d2e1f229 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -102,6 +102,9 @@ function createDummyTest() { } export function datasourceDescribe(opts: DatasourceDescribeOpts) { + // tests that call this need a lot longer timeouts + jest.setTimeout(120000) + if (process.env.DATASOURCE === "none") { createDummyTest() } diff --git a/packages/server/src/integrations/tests/utils/oracle.ts b/packages/server/src/integrations/tests/utils/oracle.ts index 8e7fd6c900..2ec8da5902 100644 --- a/packages/server/src/integrations/tests/utils/oracle.ts +++ b/packages/server/src/integrations/tests/utils/oracle.ts @@ -25,7 +25,7 @@ export async function getDatasource(): Promise { }) .withWaitStrategy( Wait.forLogMessage("DATABASE IS READY TO USE!").withStartupTimeout( - 20000 + 60000 ) ) ) From 4feda79903c592e0f58815ee982207940f60d856 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 15:52:39 +0000 Subject: [PATCH 052/192] Fix Oracle tests. --- packages/backend-core/src/sql/sql.ts | 4 +++- packages/server/src/integrations/oracle.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index baeeefbb81..c21a9b3fe5 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1743,7 +1743,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return {} } const input = this._query({ - ...json, + datasource: json.datasource, + table: json.table, + tables: json.tables, endpoint: { ...json.endpoint, operation: Operation.READ, diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index b48a95e3e6..4343593d1e 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -486,6 +486,8 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.log(query.sql, bindings) const result = await connection.execute(query.sql, bindings, options) return result as Result + } catch (err) { + throw err } finally { if (connection) { try { From 78bbf90f170435158f0220f27d6c5a4031e68570 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 16:37:29 +0000 Subject: [PATCH 053/192] Fix row.spec.ts --- .../server/src/api/routes/tests/row.spec.ts | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index b371ab1f8a..b5fff83b43 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -3548,35 +3548,16 @@ if (descriptions.length) { }) let resp = await config.api.row.search(table1._id!) - expect(resp.rows).toEqual([ - expect.objectContaining({ - _id: "%5B'1'%5D", - table1Id: "1", - many: [ - { - _id: "%5B'2'%5D", - primaryDisplay: "2", - }, - ], - tableId: table1._id, - }), - ]) + expect(resp.rows).toHaveLength(1) + expect(resp.rows[0]._id).toBe("%5B'1'%5D") + expect(resp.rows[0].many).toHaveLength(1) + expect(resp.rows[0].many[0]._id).toBe("%5B'2'%5D") resp = await config.api.row.search(table2._id!) - expect(resp.rows).toEqual([ - expect.objectContaining({ - _id: "%5B'2'%5D", - table2Id: "2", - table1Ref: "1", - one: [ - { - _id: "%5B'1'%5D", - primaryDisplay: "1", - }, - ], - tableId: table2._id, - }), - ]) + expect(resp.rows).toHaveLength(1) + expect(resp.rows[0]._id).toBe("%5B'2'%5D") + expect(resp.rows[0].one).toHaveLength(1) + expect(resp.rows[0].one[0]._id).toBe("%5B'1'%5D") }) }) } From dbe9eedcbadc6d164447078039801a53a2dcfa65 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 27 Nov 2024 17:28:20 +0000 Subject: [PATCH 054/192] Fix table.spec.ts. --- packages/backend-core/src/sql/sql.ts | 19 +++++--------- packages/backend-core/src/sql/sqlTable.ts | 8 +++--- packages/pro | 2 +- .../api/controllers/row/ExternalRequest.ts | 6 ++--- .../api/controllers/table/ExternalRequest.ts | 14 ++++++---- .../src/api/controllers/table/external.ts | 2 +- .../routes/tests/queries/generic-sql.spec.ts | 2 +- .../server/src/integrations/googlesheets.ts | 2 +- packages/server/src/integrations/oracle.ts | 2 +- .../src/integrations/tests/sqlAlias.spec.ts | 5 ++-- .../src/sdk/app/rows/search/internal/sqs.ts | 2 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 26 +++++++++++-------- .../src/sdk/app/tables/external/index.ts | 8 +++++- packages/types/src/sdk/search.ts | 7 ++--- 15 files changed, 58 insertions(+), 49 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c21a9b3fe5..d251ff672d 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,7 +3,6 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isInternalTableID, isInvalidISODateString, isValidFilter, isValidISODateString, @@ -497,9 +496,8 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const { relationships, endpoint, tableAliases: aliases } = this.query - const tableName = endpoint.entityId - const fromAlias = aliases?.[tableName] || tableName + const { relationships, endpoint, tableAliases: aliases, table } = this.query + const fromAlias = aliases?.[table.name] || table.name const matches = (value: string) => filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { @@ -1455,14 +1453,14 @@ class InternalBuilder { } qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder { - let alias = this.query.tableAliases?.[this.query.endpoint.entityId] + let alias = this.query.tableAliases?.[this.query.table.name] if (opts?.alias === false) { alias = undefined } else if (typeof opts?.alias === "string") { alias = opts.alias } return this.knex( - this.tableNameWithSchema(this.query.endpoint.entityId, { + this.tableNameWithSchema(this.query.table.name, { alias, schema: this.query.endpoint.schema, }) @@ -1558,11 +1556,10 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships } = this.query + let { endpoint, filters, paginate, relationships, table } = this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT - const tableName = endpoint.entityId // start building the query let query = this.qualifiedKnex() // handle pagination @@ -1610,9 +1607,7 @@ class InternalBuilder { // handle relationships with a CTE for all others if (relationships?.length && aggregations.length === 0) { - const mainTable = - this.query.tableAliases?.[this.query.endpoint.entityId] || - this.query.endpoint.entityId + const mainTable = this.query.tableAliases?.[table.name] || table.name const cte = this.addSorting( this.knex .with("paginated", query) @@ -1622,7 +1617,7 @@ class InternalBuilder { }) ) // add JSON aggregations attached to the CTE - return this.addJsonRelationships(cte, tableName, relationships) + return this.addJsonRelationships(cte, table.name, relationships) } return query diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index ee5dfbf453..5b76e155f9 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -25,7 +25,7 @@ function generateSchema( schema: CreateTableBuilder, table: Table, tables: Record, - oldTable: null | Table = null, + oldTable?: Table, renamed?: RenameColumn ) { let primaryKeys = table && table.primary ? table.primary : [] @@ -55,7 +55,7 @@ function generateSchema( ) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct - const oldColumn = oldTable ? oldTable.schema[key] : null + const oldColumn = oldTable?.schema[key] if ( (oldColumn && oldColumn.type) || columnTypeSet.includes(key) || @@ -199,7 +199,7 @@ function buildUpdateTable( knex: SchemaBuilder, table: Table, tables: Record, - oldTable: Table, + oldTable?: Table, renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { @@ -281,7 +281,7 @@ class SqlTableQueryBuilder { client, json.table, json.tables, - json.table, + json.meta?.oldTable, json.meta?.renamed ) diff --git a/packages/pro b/packages/pro index 06e4506cef..8c5125b4f6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 06e4506cef37110e509a0ef0b9811ac07f8b2844 +Subproject commit 8c5125b4f6e53d0a43d4de90aecbf53d9f7da8ff diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 422896cd52..5f01f323e6 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -137,7 +137,7 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { datasource: datasourceId, entityId: tableName, operation } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -706,8 +706,8 @@ export class ExternalRequest { let json: QueryJson = { endpoint: { - datasource: this.datasource, - entityId: table.name, + datasourceId: this.datasource, + entityId: table, operation, }, resource: { diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index ca4fcb1d16..df5707b072 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,19 +11,23 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, + oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { endpoint: { - datasource, - entityId: table._id!, + datasourceId: datasource, + entityId: table, operation, }, } + if (!json.meta) { + json.meta = {} + } + if (oldTable) { + json.meta.oldTable = oldTable + } if (renamed) { - if (!json.meta) { - json.meta = {} - } json.meta.renamed = renamed } return makeExternalQuery(json) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 6f09bf4a61..5e77dc1dc6 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -54,7 +54,7 @@ export async function updateTable( return table } catch (err: any) { if (err instanceof Error) { - ctx.throw(400, err.message) + throw err } else { ctx.throw(err.status || 500, err?.message || err) } diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 9130034501..44b21e0350 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -837,7 +837,7 @@ if (descriptions.length) { const res = await config.api.datasource.query({ endpoint: { - datasource: datasource._id!, + datasourceId: datasource._id!, operation: Operation.READ, entityId, }, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d8bf0e376d..6492a651e1 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -382,7 +382,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { } async query(json: EnrichedQueryJson): Promise { - const sheet = json.endpoint.entityId + const sheet = json.table.name switch (json.endpoint.operation) { case Operation.CREATE: return this.create({ sheet, row: json.body as Row }) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 4343593d1e..d690a0e17e 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -580,7 +580,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { operation !== Operation.DELETE ) { const lastRow = await this.internalQuery({ - sql: `SELECT * FROM "${json.endpoint.entityId}" WHERE ROWID = '${response.lastRowid}'`, + sql: `SELECT * FROM "${json.table.name}" WHERE ROWID = '${response.lastRowid}'`, }) return lastRow.rows as Row[] } else { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index c06ba2d35c..4de84f689e 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -231,8 +231,7 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: multiline( - `select top (@p0) * from [people] as [a] where CASE WHEN [a].[name] = @p1 THEN 1 ELSE 0 END = 1 and - CASE WHEN [a].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [a].[name] asc` + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` ), bindings: [1, "Test", 22], }) @@ -270,7 +269,7 @@ describe("Captures of real examples", () => { fields: string[] = ["a"] ): EnrichedQueryJson { return { - endpoint: { datasource: "", entityId: "", operation: op }, + endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index d6ac2a5862..ba5614edca 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -358,7 +358,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasource: SQS_DATASOURCE_INTERNAL, + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 2f9bea3c91..62e0db9aad 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -222,7 +222,7 @@ export default class AliasTables { aliases: this.aliasMap([ relationship.through, relationship.tableName, - json.endpoint.entityId, + json.table.name, ]), })) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 17bbebb251..977cdb9bd2 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -23,8 +23,6 @@ import { isSQL } from "../../../integrations/utils" import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" -import { breakExternalTableId } from "@budibase/backend-core/src/sql/utils" -import { isDatasourceId } from "@budibase/backend-core/src/docIds" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -90,18 +88,15 @@ export async function enrichQueryJson( json: QueryJson ): Promise { let datasource: Datasource | undefined = undefined - let entityId = json.endpoint.entityId - if (typeof json.endpoint.datasource === "string") { - if (json.endpoint.datasource !== SQS_DATASOURCE_INTERNAL) { - datasource = await sdk.datasources.get(json.endpoint.datasource, { + + if (typeof json.endpoint.datasourceId === "string") { + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId, { enriched: true, }) } } else { - datasource = json.endpoint.datasource - if (isDatasourceId(entityId)) { - entityId = breakExternalTableId(entityId).tableName - } + datasource = json.endpoint.datasourceId } let tables: Record @@ -111,7 +106,16 @@ export async function enrichQueryJson( tables = processInternalTables(await sdk.tables.getAllInternalTables()) } - const table = tables[entityId] + let table: Table + if (typeof json.endpoint.entityId === "string") { + let entityId = json.endpoint.entityId + if (docIds.isDatasourceId(entityId)) { + entityId = sql.utils.breakExternalTableId(entityId).tableName + } + table = tables[entityId] + } else { + table = json.endpoint.entityId + } return { table, diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 5d8f03a7a6..3511222678 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -241,7 +241,13 @@ export async function save( } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, operation, tableToSave, opts?.renaming) + await makeTableRequest( + datasource, + operation, + tableToSave, + oldTable, + opts?.renaming + ) // update any extra tables (like foreign keys in other tables) for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 72f7de455a..e25390bdd0 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -158,8 +158,8 @@ export interface ManyToManyRelationshipJson { export interface QueryJson { endpoint: { - datasource: string | Datasource - entityId: string + datasourceId: string | Datasource + entityId: string | Table operation: Operation schema?: string } @@ -173,6 +173,7 @@ export interface QueryJson { body?: Row | Row[] meta?: { renamed?: RenameColumn + oldTable?: Table // can specify something that columns could be prefixed with columnPrefix?: string } @@ -190,7 +191,7 @@ export interface EnrichedQueryJson extends QueryJson { } export interface QueryJsonRequest extends Omit { - endpoint: QueryJson["endpoint"] & { datasource: string } + endpoint: QueryJson["endpoint"] & { datasourceId: string; entityId: string } } export interface QueryOptions { From ce44975317dbc389a763883118ad52c2393de37e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 18:07:44 +0000 Subject: [PATCH 055/192] Removing test changes. --- .github/workflows/budibase_ci.yml | 1 - packages/server/src/api/routes/tests/row.spec.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 05837f217c..1258bddcca 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -27,7 +27,6 @@ env: NX_BASE_BRANCH: origin/${{ github.base_ref }} ONLY_AFFECTED_TASKS: ${{ github.event_name == 'pull_request' }} IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} - DEBUG: testcontainers* jobs: lint: diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 970e5f3619..14be60aa8c 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -85,7 +85,7 @@ function encodeJS(binding: string) { return `{{ js "${Buffer.from(binding).toString("base64")}"}}` } -const descriptions = datasourceDescribe({ only: [DatabaseName.SQS] }) +const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] }) if (descriptions.length) { describe.each(descriptions)( From b95b53f648cf87e3398ea9bb0da7fe9394470684 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 18:09:09 +0000 Subject: [PATCH 056/192] Correcting yarn lock. --- yarn.lock | 5 ----- 1 file changed, 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index edd5550b30..dd01832bba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12635,11 +12635,6 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -husky@^9.1.4: - version "9.1.7" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" - integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== - ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" From 04ea2b99e9a0ec190c37172bd7c4d397eef99cf4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Nov 2024 18:37:05 +0000 Subject: [PATCH 057/192] Fixing issue uncovered by test case. --- .../src/api/controllers/row/staticFormula.ts | 17 ++++++++++++++++- .../server/src/api/routes/tests/row.spec.ts | 17 ++--------------- .../rowProcessor/bbReferenceProcessor.ts | 4 ++-- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index b9751b1ce2..ff2381e61c 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -19,6 +19,17 @@ import { cloneDeep, merge } from "lodash/fp" import sdk from "../../../sdk" import * as pro from "@budibase/pro" +function mergeRows(row1: Row, row2: Row) { + const merged = merge(row1, row2) + // make sure any specifically undefined fields are removed + for (const key of Object.keys(row2)) { + if (row2[key] === undefined) { + delete merged[key] + } + } + return merged +} + /** * This function runs through a list of enriched rows, looks at the rows which * are related and then checks if they need the state of their formulas @@ -164,7 +175,11 @@ export async function finaliseRow( await db.put(row) const retrieved = await db.tryGet(row._id) - enrichedRow = merge(retrieved, enrichedRow) + if (!retrieved) { + throw new Error(`Unable to retrieve row ${row._id} after saving.`) + } + + enrichedRow = mergeRows(retrieved, enrichedRow) enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, }) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 14be60aa8c..25c9b8abef 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -2571,14 +2571,12 @@ if (descriptions.length) { let tableId: string let o2mData: Row[] let m2mData: Row[] - let isRelationship: boolean beforeAll(async () => { const table = await config.api.table.save( defaultTable({ schema: relSchema() }) ) tableId = table._id! - isRelationship = relSchema().user.type === FieldType.LINK o2mData = [ await dataGenerator(o2mTable._id!), @@ -2755,19 +2753,8 @@ if (descriptions.length) { user: null, users: null, }) - expect(updatedRow).toEqual({ - name: "foo", - description: "bar", - tableId, - _id: row._id, - _rev: expect.any(String), - id: isInternal ? undefined : expect.any(Number), - type: isInternal ? "row" : undefined, - createdAt: isInternal ? new Date().toISOString() : undefined, - updatedAt: isInternal ? new Date().toISOString() : undefined, - users: isRelationship ? undefined : [], - user: isRelationship ? undefined : [], - }) + expect(updatedRow.user).toBeUndefined() + expect(updatedRow.users).toBeUndefined() }) it("fetch all will populate the relationships", async () => { diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index b02ea2ff60..6c8ecc3167 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -153,10 +153,10 @@ export async function processOutputBBReference( } export async function processOutputBBReferences( - value: string | null | undefined, + value: string | string[] | null | undefined, subtype: BBReferenceFieldSubType ): Promise { - if (!value) { + if (!value || (Array.isArray(value) && value.length === 0)) { return undefined } const ids = From f49f9280d9c676956439449884dec0c9dcf5b5e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 09:42:01 +0000 Subject: [PATCH 058/192] Working on search tests. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/integrations/oracle.ts | 2 -- packages/server/src/sdk/app/tables/external/index.ts | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5384444067..6a56ec37c6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe("primaryDisplay", () => { + describe.only("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index d690a0e17e..ded21487f6 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -486,8 +486,6 @@ class OracleIntegration extends Sql implements DatasourcePlus { this.log(query.sql, bindings) const result = await connection.execute(query.sql, bindings, options) return result as Result - } catch (err) { - throw err } finally { if (connection) { try { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 3511222678..a8fad40a8b 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -252,7 +252,11 @@ export async function save( for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, op, extraTable) + try { + await makeTableRequest(datasource, op, extraTable) + } catch (err: any) { + throw err + } } // make sure the constrained list, all still exist From b700f16a3a6f1ed00b412ea2ff88a96598332f24 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 10:12:16 +0000 Subject: [PATCH 059/192] Fix search.spec.ts. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/sdk/app/tables/external/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6a56ec37c6..5384444067 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe.only("primaryDisplay", () => { + describe("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index a8fad40a8b..df3f287842 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -253,7 +253,7 @@ export async function save( const oldExtraTable = oldTables[extraTable.name] let op = oldExtraTable ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE try { - await makeTableRequest(datasource, op, extraTable) + await makeTableRequest(datasource, op, extraTable, oldExtraTable) } catch (err: any) { throw err } From 97b1c41313ce88a3a00eddf6981c11038b71590f Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 11:36:59 +0000 Subject: [PATCH 060/192] update openai automation tests --- packages/server/src/automations/actions.ts | 5 ++ .../src/automations/tests/openai.spec.ts | 71 ++++++++++++++----- .../tests/utilities/AutomationTestBuilder.ts | 13 ++++ .../app/automation/StepInputsOutputs.ts | 2 +- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a8b69fa7d7..edac2ed690 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -96,6 +96,11 @@ if (env.SELF_HOSTED) { ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + + if (env.isTest()) { + ACTION_IMPLS["OPENAI"] = openai.run + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + } } export async function getActionDefinitions() { diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 8119750f8b..a4d98fc9d3 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -1,7 +1,9 @@ -import { getConfig, runStep, afterAll as _afterAll } from "./utilities" +import { getConfig, afterAll as _afterAll } from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import { OpenAI } from "openai" import { setEnv as setCoreEnv } from "@budibase/backend-core" import * as pro from "@budibase/pro" +import { Model } from "@budibase/types" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -47,6 +49,7 @@ describe("test the openai action", () => { let resetEnv: () => void | undefined beforeAll(async () => { + setCoreEnv({ SELF_HOSTED: true }) await config.init() }) @@ -62,17 +65,39 @@ describe("test the openai action", () => { afterAll(_afterAll) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await runStep(config, "OPENAI", { prompt: OPENAI_PROMPT }) - expect(res.response).toEqual("This is a test") - expect(res.success).toBeTruthy() + setCoreEnv({ SELF_HOSTED: true }) + + const result = await createAutomationBuilder({ + name: "Test OpenAI Response", + config, + }) + .appAction({ fields: {} }) + .openai( + { prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI }, + { stepName: "Basic OpenAI Query" } + ) + .run() + + expect(result.steps[0].outputs.response).toEqual("This is a test") + expect(result.steps[0].outputs.success).toBeTruthy() }) it("should present the correct error message when a prompt is not provided", async () => { - const res = await runStep(config, "OPENAI", { prompt: null }) - expect(res.response).toEqual( + const result = await createAutomationBuilder({ + name: "Test OpenAI No Prompt", + config, + }) + .appAction({ fields: {} }) + .openai( + { prompt: "", model: Model.GPT_4O_MINI }, + { stepName: "Empty Prompt Query" } + ) + .run() + + expect(result.steps[0].outputs.response).toEqual( "Budibase OpenAI Automation Failed: No prompt supplied" ) - expect(res.success).toBeFalsy() + expect(result.steps[0].outputs.success).toBeFalsy() }) it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { @@ -91,14 +116,21 @@ describe("test the openai action", () => { } as any) ) - const res = await runStep(config, "OPENAI", { - prompt: OPENAI_PROMPT, + const result = await createAutomationBuilder({ + name: "Test OpenAI Error", + config, }) + .appAction({ fields: {} }) + .openai( + { prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI }, + { stepName: "Error Producing Query" } + ) + .run() - expect(res.response).toEqual( + expect(result.steps[0].outputs.response).toEqual( "Error: An error occurred while calling createChatCompletion" ) - expect(res.success).toBeFalsy() + expect(result.steps[0].outputs.success).toBeFalsy() }) it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { @@ -106,10 +138,19 @@ describe("test the openai action", () => { jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) const prompt = "What is the meaning of life?" - await runStep(config, "OPENAI", { - model: "gpt-4o-mini", - prompt, + await createAutomationBuilder({ + name: "Test OpenAI Pro Features", + config, }) + .appAction({ fields: {} }) + .openai( + { + model: Model.GPT_4O_MINI, + prompt, + }, + { stepName: "Pro Features Query" } + ) + .run() expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( "gpt-4o-mini" @@ -117,8 +158,6 @@ describe("test the openai action", () => { const llmInstance = mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value - // init does not appear to be called currently - // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 78830adf2c..0c091ea00b 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -35,6 +35,7 @@ import { Branch, FilterStepInputs, ExecuteScriptStepInputs, + OpenAIStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -221,6 +222,18 @@ class BaseStepBuilder { input ) } + + openai( + input: OpenAIStepInputs, + opts?: { stepName?: string; stepId?: string } + ): this { + return this.step( + AutomationActionStepId.OPENAI, + BUILTIN_ACTION_DEFINITIONS.OPENAI, + input, + opts + ) + } } class StepBuilder extends BaseStepBuilder { build(): AutomationStep[] { diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts index 3aadb77108..b2f679edee 100644 --- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts +++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts @@ -150,7 +150,7 @@ export type OpenAIStepInputs = { prompt: string model: Model } -enum Model { +export enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API GPT_4 = "gpt-4", From 3ef854085702fb012540bf28644f1d754e2efe79 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 11:52:00 +0000 Subject: [PATCH 061/192] migrate queryrows --- .../src/automations/tests/queryRows.spec.ts | 200 ++++++++++-------- 1 file changed, 114 insertions(+), 86 deletions(-) diff --git a/packages/server/src/automations/tests/queryRows.spec.ts b/packages/server/src/automations/tests/queryRows.spec.ts index 12611d3f90..18d2e2d6cd 100644 --- a/packages/server/src/automations/tests/queryRows.spec.ts +++ b/packages/server/src/automations/tests/queryRows.spec.ts @@ -1,5 +1,7 @@ -import { Table } from "@budibase/types" +import { EmptyFilterOption, SortOrder, Table } from "@budibase/types" import * as setup from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" +import * as automation from "../index" const NAME = "Test" @@ -8,6 +10,7 @@ describe("Test a query step automation", () => { let config = setup.getConfig() beforeAll(async () => { + await automation.init() await config.init() table = await config.createTable() const row = { @@ -22,107 +25,132 @@ describe("Test a query step automation", () => { afterAll(setup.afterAll) it("should be able to run the query step", async () => { - const inputs = { - tableId: table._id, - filters: { - equal: { - name: NAME, - }, - }, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Basic Query Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) - expect(res.rows[0].name).toBe(NAME) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: { + equal: { + name: NAME, + }, + }, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query All Rows" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) + expect(result.steps[0].outputs.rows[0].name).toBe(NAME) }) it("Returns all rows when onEmptyFilter has no value and no filters are passed", async () => { - const inputs = { - tableId: table._id, - filters: {}, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Empty Filter Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) - expect(res.rows[0].name).toBe(NAME) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Empty Filter" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) + expect(result.steps[0].outputs.rows[0].name).toBe(NAME) }) it("Returns no rows when onEmptyFilter is RETURN_NONE and theres no filters", async () => { - const inputs = { - tableId: table._id, - filters: {}, - "filters-def": [], - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - onEmptyFilter: "none", - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Return None Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(false) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(0) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + "filters-def": [], + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + { stepName: "Query With Return None" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(false) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(0) }) it("Returns no rows when onEmptyFilters RETURN_NONE and a filter is passed with a null value", async () => { - const inputs = { - tableId: table._id, - onEmptyFilter: "none", - filters: {}, - "filters-def": [ - { - value: null, - }, - ], - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Null Filter Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(false) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(0) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + filters: {}, + "filters-def": [ + { + value: null, + }, + ], + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Null Filter" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(false) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(0) }) it("Returns rows when onEmptyFilter is RETURN_ALL and no filter is passed", async () => { - const inputs = { - tableId: table._id, - onEmptyFilter: "all", - filters: {}, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Return All Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + filters: {}, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Return All" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) }) }) From 88773d1e6cb13beb41fa68950b50d7490aed9d70 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 13:42:54 +0100 Subject: [PATCH 062/192] Clean unused scripts --- package.json | 1 - scripts/buildx-multiarch.sh | 4 --- scripts/pinVersions.js | 46 ------------------------ scripts/releaseHelmChart.js | 28 --------------- scripts/resetVersions.sh | 7 ---- scripts/syncLocalDependencies.js | 60 -------------------------------- scripts/updateVersions.sh | 8 ----- 7 files changed, 154 deletions(-) delete mode 100755 scripts/buildx-multiarch.sh delete mode 100644 scripts/pinVersions.js delete mode 100755 scripts/releaseHelmChart.js delete mode 100755 scripts/resetVersions.sh delete mode 100755 scripts/syncLocalDependencies.js delete mode 100755 scripts/updateVersions.sh diff --git a/package.json b/package.json index e354f36d2a..31e4110a35 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,6 @@ "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.3.3 -t budibase/couchdb:v3.3.3-sqs-v2.1.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", - "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run --stream env:multi:enable", "env:multi:disable": "lerna run --stream env:multi:disable", "env:selfhost:enable": "lerna run --stream env:selfhost:enable", diff --git a/scripts/buildx-multiarch.sh b/scripts/buildx-multiarch.sh deleted file mode 100755 index 07a8fdb5c1..0000000000 --- a/scripts/buildx-multiarch.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -sudo apt-get install -y qemu qemu-user-static -docker buildx create --name budibase -docker buildx use budibase diff --git a/scripts/pinVersions.js b/scripts/pinVersions.js deleted file mode 100644 index 44cbba1090..0000000000 --- a/scripts/pinVersions.js +++ /dev/null @@ -1,46 +0,0 @@ -const fs = require("fs") -const path = require("path") - -const MONOREPO_ROOT = "packages" - -const packages = getPackages() - -function getPackages() { - if (fs.existsSync(MONOREPO_ROOT)) { - return fs.readdirSync(MONOREPO_ROOT).map(pkg => path.join(MONOREPO_ROOT, pkg)) - } else { - return ["./"] - } -} - -function pinDeps(dependencies) { - for (let dependency in dependencies) { - if (dependency.startsWith("@budibase")) { - dependencies[dependency] = dependencies[dependency].replace("^", "") - } - } - return dependencies -} - -// iterate over the monorepo packages -for (let pkgPath of packages) { - // only directories - if (fs.statSync(pkgPath).isDirectory()) { - // get the package JSON file - const pkgJsonPath = path.join(pkgPath, "package.json") - if (!fs.existsSync(pkgJsonPath)) { - continue - } - const pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath)) - - - // find any budibase dependencies, and pin them - pkgJson.dependencies = pinDeps(pkgJson.dependencies) - pkgJson.devDependencies = pinDeps(pkgJson.devDependencies) - - // update the package JSON files - fs.writeFileSync(pkgJsonPath, JSON.stringify(pkgJson, null, 2)) - } -} - -console.log("Pinned dev versions for budibase packages successfully.") diff --git a/scripts/releaseHelmChart.js b/scripts/releaseHelmChart.js deleted file mode 100755 index 45ae01df0e..0000000000 --- a/scripts/releaseHelmChart.js +++ /dev/null @@ -1,28 +0,0 @@ -const yaml = require("js-yaml") -const fs = require("fs") -const path = require("path") - -const CHART_PATH = path.join(__dirname, "../", "charts", "budibase", "Chart.yaml") -const UPGRADE_VERSION = process.env.BUDIBASE_RELEASE_VERSION - -if (!UPGRADE_VERSION) { - throw new Error("BUDIBASE_RELEASE_VERSION env var must be set.") -} - -try { - const chartFile = fs.readFileSync(CHART_PATH, "utf-8") - const chart = yaml.load(chartFile) - - // Upgrade app version in chart to match budibase release version - chart.appVersion = UPGRADE_VERSION - - // semantically version the chart - const [major, minor, patch] = chart.version.split(".") - const newPatch = parseInt(patch) + 1 - chart.version = [major, minor, newPatch].join(".") - const updatedChartYaml = yaml.dump(chart) - fs.writeFileSync(CHART_PATH, updatedChartYaml) -} catch (err) { - console.error("Error releasing helm chart") - throw err -} diff --git a/scripts/resetVersions.sh b/scripts/resetVersions.sh deleted file mode 100755 index 83411f5153..0000000000 --- a/scripts/resetVersions.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -echo "Resetting package versions" -yarn lerna exec "yarn version --no-git-tag-version --new-version=0.0.0" -echo "Updating dependencies" -node scripts/syncLocalDependencies.js "0.0.0" -git checkout package.json -echo "Package versions reset!" diff --git a/scripts/syncLocalDependencies.js b/scripts/syncLocalDependencies.js deleted file mode 100755 index 1c402e9a44..0000000000 --- a/scripts/syncLocalDependencies.js +++ /dev/null @@ -1,60 +0,0 @@ -const fs = require("fs") -const path = require("path") -const { execSync } = require("child_process") - -// Get the version argument from the command line -const version = process.argv[2] -if (!version) { - console.error("Usage: node update-workspace-dependencies.js ") - process.exit(1) -} - -// Get the list of workspaces with mismatched dependencies -const output = execSync("yarn --silent workspaces info --json", { - encoding: "utf-8", -}) -const data = JSON.parse(output) - -const workspaces = Object.keys(data).filter(key => { - return data[key].mismatchedWorkspaceDependencies?.length -}) - -// Loop through each workspace and update the dependencies -workspaces.forEach(workspace => { - const dependencies = data[workspace].mismatchedWorkspaceDependencies - - // Loop through each dependency and update its version in package.json - const packageJsonPath = path.join(data[workspace].location, "package.json") - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8")) - let hasChanges = false - - dependencies.forEach(dependency => { - if (packageJson.dependencies?.[dependency]) { - packageJson.dependencies[dependency] = version - hasChanges = true - } - if (packageJson.devDependencies?.[dependency]) { - packageJson.devDependencies[dependency] = version - hasChanges = true - } - if (packageJson.peerDependencies?.[dependency]) { - packageJson.peerDependencies[dependency] = version - hasChanges = true - } - }) - - // Write changes to package.json if there are any - if (hasChanges) { - fs.writeFileSync( - packageJsonPath, - JSON.stringify(packageJson, null, 2) + "\n" - ) - } -}) - -const rootPackageJson = JSON.parse(fs.readFileSync("package.json", "utf-8")) -delete rootPackageJson["resolutions"] -fs.writeFileSync( - "package.json", - JSON.stringify(rootPackageJson, null, 2) + "\n" -) diff --git a/scripts/updateVersions.sh b/scripts/updateVersions.sh deleted file mode 100755 index 1d019fd886..0000000000 --- a/scripts/updateVersions.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -version=$(./scripts/getCurrentVersion.sh) -echo "Setting version $version" -yarn lerna exec "yarn version --no-git-tag-version --new-version=$version" -echo "Updating dependencies" -node scripts/syncLocalDependencies.js $version -echo "Syncing yarn workspace" -yarn From 20df29d78120b37f91ac7d4870b719b6dfb1731b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 14:19:45 +0100 Subject: [PATCH 063/192] Update yarn.lock for bb packages --- yarn.lock | 157 ++++++++++++++---------------------------------------- 1 file changed, 40 insertions(+), 117 deletions(-) diff --git a/yarn.lock b/yarn.lock index 90e41f0084..2e7a743310 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2196,7 +2196,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@3.2.11": +"@budibase/backend-core@3.2.14": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2280,18 +2280,18 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.2.11" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.11.tgz#40d4929b3958dacca3f4c207718a4647c08a6100" - integrity sha512-xE1tx/C2cnbyR4s/6XkkweoQw6CW5fsQt++gzrrML8abgsODru+tA7M2NbWlKsEnVWHDQRvUVcXm4wqDeNNZ9g== + version "3.2.14" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.14.tgz#2ea1ef4372803304ac3a06ce77620ce54a09dd02" + integrity sha512-ski7NeXOdDIjC0vapo+hxr8/VnbDhmOLF5TdvoLBgviMk7ij2rNPI2dNs+6kQuVXAEVJPD6bJ4Ke0z2d5YcYrQ== dependencies: "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "3.2.11" - "@budibase/shared-core" "3.2.11" - "@budibase/string-templates" "3.2.11" - "@budibase/types" "3.2.11" + "@budibase/backend-core" "3.2.14" + "@budibase/shared-core" "3.2.14" + "@budibase/string-templates" "3.2.14" + "@budibase/types" "3.2.14" "@koa/router" "13.1.0" bull "4.10.1" - dd-trace "5.23.0" + dd-trace "5.26.0" joi "17.6.0" jsonwebtoken "9.0.2" lru-cache "^7.14.1" @@ -2301,13 +2301,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@3.2.11": +"@budibase/shared-core@3.2.14": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@3.2.11": +"@budibase/string-templates@3.2.14": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2315,7 +2315,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@3.2.11": +"@budibase/types@3.2.14": version "0.0.0" dependencies: scim-patch "^0.8.1" @@ -2482,13 +2482,6 @@ resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.2.2.tgz#ac02c76ac9a38250dca740727c7cdf00244ce3d3" integrity sha512-rTWo96mEPTY5UbtGoFj8/wY0uKSViJhsPg/Z6aoFWBFXQ8b45Ix2e/yvf92AAwrhG+gPLTxEqTXh3kef2dP8Ow== -"@datadog/native-appsec@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.1.1.tgz#76aa34697e6ecbd3d9ef7e6938d3cdcfa689b1f3" - integrity sha512-mf+Ym/AzET4FeUTXOs8hz0uLOSsVIUnavZPUx8YoKWK5lKgR2L+CLfEzOpjBwgFpDgbV8I1/vyoGelgGpsMKHA== - dependencies: - node-gyp-build "^3.9.0" - "@datadog/native-appsec@8.3.0": version "8.3.0" resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.3.0.tgz#91afd89d18d386be4da8a1b0e04500f2f8b5eb66" @@ -2496,14 +2489,6 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-iast-rewriter@2.4.1": - version "2.4.1" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.4.1.tgz#e8211f78c818906513fb96a549374da0382c7623" - integrity sha512-j3auTmyyn63e2y+SL28CGNy/l+jXQyh+pxqoGTacWaY5FW/dvo5nGQepAismgJ3qJ8VhQfVWRdxBSiT7wu9clw== - dependencies: - lru-cache "^7.14.0" - node-gyp-build "^4.5.0" - "@datadog/native-iast-rewriter@2.5.0": version "2.5.0" resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.5.0.tgz#b613defe86e78168f750d1f1662d4ffb3cf002e6" @@ -2512,13 +2497,6 @@ lru-cache "^7.14.0" node-gyp-build "^4.5.0" -"@datadog/native-iast-taint-tracking@3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.1.0.tgz#7b2ed7f8fad212d65e5ab03bcdea8b42a3051b2e" - integrity sha512-rw6qSjmxmu1yFHVvZLXFt/rVq2tUZXocNogPLB8n7MPpA0jijNGb109WokWw5ITImiW91GcGDuBW6elJDVKouQ== - dependencies: - node-gyp-build "^3.9.0" - "@datadog/native-iast-taint-tracking@3.2.0": version "3.2.0" resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.2.0.tgz#9fb6823d82f934e12c06ea1baa7399ca80deb2ec" @@ -2526,14 +2504,6 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-metrics@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-2.0.0.tgz#65bf03313ee419956361e097551db36173e85712" - integrity sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA== - dependencies: - node-addon-api "^6.1.0" - node-gyp-build "^3.9.0" - "@datadog/native-metrics@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-3.0.1.tgz#dc276c93785c0377a048e316f23b7c8ff3acfa84" @@ -2542,17 +2512,6 @@ node-addon-api "^6.1.0" node-gyp-build "^3.9.0" -"@datadog/pprof@5.3.0": - version "5.3.0" - resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.3.0.tgz#c2f58d328ecced7f99887f1a559d7fe3aecb9219" - integrity sha512-53z2Q3K92T6Pf4vz4Ezh8kfkVEvLzbnVqacZGgcbkP//q0joFzO8q00Etw1S6NdnCX0XmX08ULaF4rUI5r14mw== - dependencies: - delay "^5.0.0" - node-gyp-build "<4.0" - p-limit "^3.1.0" - pprof-format "^2.1.0" - source-map "^0.7.4" - "@datadog/pprof@5.4.1": version "5.4.1" resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.4.1.tgz#08c9bcf5d8efb2eeafdfc9f5bb5402f79fb41266" @@ -3469,16 +3428,6 @@ resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== -"@jsep-plugin/assignment@^1.2.1": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@jsep-plugin/assignment/-/assignment-1.3.0.tgz#fcfc5417a04933f7ceee786e8ab498aa3ce2b242" - integrity sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ== - -"@jsep-plugin/regex@^1.0.3": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@jsep-plugin/regex/-/regex-1.0.4.tgz#cb2fc423220fa71c609323b9ba7f7d344a755fcc" - integrity sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg== - "@koa/cors@5.0.0": version "5.0.0" resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd" @@ -9321,43 +9270,6 @@ dc-polyfill@^0.1.4: resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.6.tgz#c2940fa68ffb24a7bf127cc6cfdd15b39f0e7f02" integrity sha512-UV33cugmCC49a5uWAApM+6Ev9ZdvIUMTrtCO9fj96TPGOQiea54oeO3tiEVdVeo3J9N2UdJEmbS4zOkkEA35uQ== -dd-trace@5.23.0: - version "5.23.0" - resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.23.0.tgz#a0c11863406de440a6675648caf06e1d07d67ba8" - integrity sha512-nLvwSGpTMIk6S3sMSge6yFqqgqI573VgZc8MF31vl6K0ouJoE7OkVx9cmSVjS4CbSi525tcKq9z7tApsNLpVLQ== - dependencies: - "@datadog/native-appsec" "8.1.1" - "@datadog/native-iast-rewriter" "2.4.1" - "@datadog/native-iast-taint-tracking" "3.1.0" - "@datadog/native-metrics" "^2.0.0" - "@datadog/pprof" "5.3.0" - "@datadog/sketches-js" "^2.1.0" - "@opentelemetry/api" ">=1.0.0 <1.9.0" - "@opentelemetry/core" "^1.14.0" - crypto-randomuuid "^1.0.0" - dc-polyfill "^0.1.4" - ignore "^5.2.4" - import-in-the-middle "1.11.2" - int64-buffer "^0.1.9" - istanbul-lib-coverage "3.2.0" - jest-docblock "^29.7.0" - jsonpath-plus "^9.0.0" - koalas "^1.0.2" - limiter "1.1.5" - lodash.sortby "^4.7.0" - lru-cache "^7.14.0" - module-details-from-path "^1.0.3" - msgpack-lite "^0.1.26" - opentracing ">=0.12.1" - path-to-regexp "^0.1.10" - pprof-format "^2.1.0" - protobufjs "^7.2.5" - retry "^0.13.1" - rfdc "^1.3.1" - semver "^7.5.4" - shell-quote "^1.8.1" - tlhunter-sorted-set "^0.1.0" - dd-trace@5.26.0: version "5.26.0" resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.26.0.tgz#cc55061f66742bf01d0d7dc9f75c0e4937c82f40" @@ -14128,11 +14040,6 @@ jsdom@^21.1.1: ws "^8.13.0" xml-name-validator "^4.0.0" -jsep@^1.3.8: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsep/-/jsep-1.4.0.tgz#19feccbfa51d8a79f72480b4b8e40ce2e17152f0" - integrity sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw== - jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -14236,15 +14143,6 @@ jsonparse@^1.2.0, jsonparse@^1.3.1: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== -jsonpath-plus@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/jsonpath-plus/-/jsonpath-plus-9.0.0.tgz#bb8703ee481531142bca8dee9a42fe72b8358a7f" - integrity sha512-bqE77VIDStrOTV/czspZhTn+o27Xx9ZJRGVkdVShEtPoqsIx5yALv3lWVU6y+PqYvWPJNWE7ORCQheQkEe0DDA== - dependencies: - "@jsep-plugin/assignment" "^1.2.1" - "@jsep-plugin/regex" "^1.0.3" - jsep "^1.3.8" - jsonschema@1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" @@ -20226,7 +20124,16 @@ string-range@~1.2, string-range@~1.2.1: resolved "https://registry.yarnpkg.com/string-range/-/string-range-1.2.2.tgz#a893ed347e72299bc83befbbf2a692a8d239d5dd" integrity sha512-tYft6IFi8SjplJpxCUxyqisD3b+R2CSkomrtJYCkvuf1KuCAWgz7YXt4O0jip7efpfCemwHEzTEAO8EuOYgh3w== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20318,7 +20225,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20332,6 +20239,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22153,7 +22067,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22171,6 +22085,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 8e2b916343ca2340f26a5dca8519fcd4f28cbf2c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 14:31:22 +0000 Subject: [PATCH 064/192] Update yarn.lock --- yarn.lock | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/yarn.lock b/yarn.lock index 85d8b160f5..edf81d1f1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4075,7 +4075,7 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== -"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0": +"@rollup/plugin-commonjs@^16.0.0": version "16.0.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f" integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw== @@ -4153,7 +4153,7 @@ is-module "^1.0.0" resolve "^1.22.1" -"@rollup/plugin-replace@^5.0.2", "@rollup/plugin-replace@^5.0.3": +"@rollup/plugin-replace@^5.0.3": version "5.0.7" resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.7.tgz#150c9ee9db8031d9e4580a61a0edeaaed3d37687" integrity sha512-PqxSfuorkHz/SPpyngLyg5GCEkOcee9M1bkxiVDr41Pd61mqP1PLOoDPbpl44SB2mQGKwV/In74gqQmGITOhEQ== @@ -12615,11 +12615,6 @@ husky@^8.0.3: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -husky@^9.1.4: - version "9.1.7" - resolved "https://registry.yarnpkg.com/husky/-/husky-9.1.7.tgz#d46a38035d101b46a70456a850ff4201344c0b2d" - integrity sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA== - ical-generator@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/ical-generator/-/ical-generator-4.1.0.tgz#2a336c951864c5583a2aa715d16f2edcdfd2d90b" @@ -20231,7 +20226,16 @@ string-range@~1.2, string-range@~1.2.1: resolved "https://registry.yarnpkg.com/string-range/-/string-range-1.2.2.tgz#a893ed347e72299bc83befbbf2a692a8d239d5dd" integrity sha512-tYft6IFi8SjplJpxCUxyqisD3b+R2CSkomrtJYCkvuf1KuCAWgz7YXt4O0jip7efpfCemwHEzTEAO8EuOYgh3w== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20323,7 +20327,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20337,6 +20341,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22158,7 +22169,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22176,6 +22187,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 1e01f0d7575b5d66f0b18338d80c4bcf626f6d8d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:03:58 +0100 Subject: [PATCH 065/192] Update yarn.lock --- yarn.lock | 102 ------------------------------------------------------ 1 file changed, 102 deletions(-) diff --git a/yarn.lock b/yarn.lock index ceebc9ebdf..70d3c2d4b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2483,13 +2483,6 @@ resolved "https://registry.yarnpkg.com/@datadog/libdatadog/-/libdatadog-0.2.2.tgz#ac02c76ac9a38250dca740727c7cdf00244ce3d3" integrity sha512-rTWo96mEPTY5UbtGoFj8/wY0uKSViJhsPg/Z6aoFWBFXQ8b45Ix2e/yvf92AAwrhG+gPLTxEqTXh3kef2dP8Ow== -"@datadog/native-appsec@8.1.1": - version "8.1.1" - resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.1.1.tgz#76aa34697e6ecbd3d9ef7e6938d3cdcfa689b1f3" - integrity sha512-mf+Ym/AzET4FeUTXOs8hz0uLOSsVIUnavZPUx8YoKWK5lKgR2L+CLfEzOpjBwgFpDgbV8I1/vyoGelgGpsMKHA== - dependencies: - node-gyp-build "^3.9.0" - "@datadog/native-appsec@8.3.0": version "8.3.0" resolved "https://registry.yarnpkg.com/@datadog/native-appsec/-/native-appsec-8.3.0.tgz#91afd89d18d386be4da8a1b0e04500f2f8b5eb66" @@ -2497,14 +2490,6 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-iast-rewriter@2.4.1": - version "2.4.1" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.4.1.tgz#e8211f78c818906513fb96a549374da0382c7623" - integrity sha512-j3auTmyyn63e2y+SL28CGNy/l+jXQyh+pxqoGTacWaY5FW/dvo5nGQepAismgJ3qJ8VhQfVWRdxBSiT7wu9clw== - dependencies: - lru-cache "^7.14.0" - node-gyp-build "^4.5.0" - "@datadog/native-iast-rewriter@2.5.0": version "2.5.0" resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.5.0.tgz#b613defe86e78168f750d1f1662d4ffb3cf002e6" @@ -2513,13 +2498,6 @@ lru-cache "^7.14.0" node-gyp-build "^4.5.0" -"@datadog/native-iast-taint-tracking@3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.1.0.tgz#7b2ed7f8fad212d65e5ab03bcdea8b42a3051b2e" - integrity sha512-rw6qSjmxmu1yFHVvZLXFt/rVq2tUZXocNogPLB8n7MPpA0jijNGb109WokWw5ITImiW91GcGDuBW6elJDVKouQ== - dependencies: - node-gyp-build "^3.9.0" - "@datadog/native-iast-taint-tracking@3.2.0": version "3.2.0" resolved "https://registry.yarnpkg.com/@datadog/native-iast-taint-tracking/-/native-iast-taint-tracking-3.2.0.tgz#9fb6823d82f934e12c06ea1baa7399ca80deb2ec" @@ -2527,14 +2505,6 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-metrics@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-2.0.0.tgz#65bf03313ee419956361e097551db36173e85712" - integrity sha512-YklGVwUtmKGYqFf1MNZuOHvTYdKuR4+Af1XkWcMD8BwOAjxmd9Z+97328rCOY8TFUJzlGUPaXzB8j2qgG/BMwA== - dependencies: - node-addon-api "^6.1.0" - node-gyp-build "^3.9.0" - "@datadog/native-metrics@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@datadog/native-metrics/-/native-metrics-3.0.1.tgz#dc276c93785c0377a048e316f23b7c8ff3acfa84" @@ -2543,17 +2513,6 @@ node-addon-api "^6.1.0" node-gyp-build "^3.9.0" -"@datadog/pprof@5.3.0": - version "5.3.0" - resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.3.0.tgz#c2f58d328ecced7f99887f1a559d7fe3aecb9219" - integrity sha512-53z2Q3K92T6Pf4vz4Ezh8kfkVEvLzbnVqacZGgcbkP//q0joFzO8q00Etw1S6NdnCX0XmX08ULaF4rUI5r14mw== - dependencies: - delay "^5.0.0" - node-gyp-build "<4.0" - p-limit "^3.1.0" - pprof-format "^2.1.0" - source-map "^0.7.4" - "@datadog/pprof@5.4.1": version "5.4.1" resolved "https://registry.yarnpkg.com/@datadog/pprof/-/pprof-5.4.1.tgz#08c9bcf5d8efb2eeafdfc9f5bb5402f79fb41266" @@ -3470,16 +3429,6 @@ resolved "https://registry.yarnpkg.com/@jsdevtools/ono/-/ono-7.1.3.tgz#9df03bbd7c696a5c58885c34aa06da41c8543796" integrity sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg== -"@jsep-plugin/assignment@^1.2.1": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@jsep-plugin/assignment/-/assignment-1.3.0.tgz#fcfc5417a04933f7ceee786e8ab498aa3ce2b242" - integrity sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ== - -"@jsep-plugin/regex@^1.0.3": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@jsep-plugin/regex/-/regex-1.0.4.tgz#cb2fc423220fa71c609323b9ba7f7d344a755fcc" - integrity sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg== - "@koa/cors@5.0.0": version "5.0.0" resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd" @@ -9311,43 +9260,6 @@ dc-polyfill@^0.1.4: resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.6.tgz#c2940fa68ffb24a7bf127cc6cfdd15b39f0e7f02" integrity sha512-UV33cugmCC49a5uWAApM+6Ev9ZdvIUMTrtCO9fj96TPGOQiea54oeO3tiEVdVeo3J9N2UdJEmbS4zOkkEA35uQ== -dd-trace@5.23.0: - version "5.23.0" - resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.23.0.tgz#a0c11863406de440a6675648caf06e1d07d67ba8" - integrity sha512-nLvwSGpTMIk6S3sMSge6yFqqgqI573VgZc8MF31vl6K0ouJoE7OkVx9cmSVjS4CbSi525tcKq9z7tApsNLpVLQ== - dependencies: - "@datadog/native-appsec" "8.1.1" - "@datadog/native-iast-rewriter" "2.4.1" - "@datadog/native-iast-taint-tracking" "3.1.0" - "@datadog/native-metrics" "^2.0.0" - "@datadog/pprof" "5.3.0" - "@datadog/sketches-js" "^2.1.0" - "@opentelemetry/api" ">=1.0.0 <1.9.0" - "@opentelemetry/core" "^1.14.0" - crypto-randomuuid "^1.0.0" - dc-polyfill "^0.1.4" - ignore "^5.2.4" - import-in-the-middle "1.11.2" - int64-buffer "^0.1.9" - istanbul-lib-coverage "3.2.0" - jest-docblock "^29.7.0" - jsonpath-plus "^9.0.0" - koalas "^1.0.2" - limiter "1.1.5" - lodash.sortby "^4.7.0" - lru-cache "^7.14.0" - module-details-from-path "^1.0.3" - msgpack-lite "^0.1.26" - opentracing ">=0.12.1" - path-to-regexp "^0.1.10" - pprof-format "^2.1.0" - protobufjs "^7.2.5" - retry "^0.13.1" - rfdc "^1.3.1" - semver "^7.5.4" - shell-quote "^1.8.1" - tlhunter-sorted-set "^0.1.0" - dd-trace@5.26.0: version "5.26.0" resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-5.26.0.tgz#cc55061f66742bf01d0d7dc9f75c0e4937c82f40" @@ -14100,11 +14012,6 @@ jsdom@^21.1.1: ws "^8.13.0" xml-name-validator "^4.0.0" -jsep@^1.3.8: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsep/-/jsep-1.4.0.tgz#19feccbfa51d8a79f72480b4b8e40ce2e17152f0" - integrity sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw== - jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -14208,15 +14115,6 @@ jsonparse@^1.2.0, jsonparse@^1.3.1: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== -jsonpath-plus@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/jsonpath-plus/-/jsonpath-plus-9.0.0.tgz#bb8703ee481531142bca8dee9a42fe72b8358a7f" - integrity sha512-bqE77VIDStrOTV/czspZhTn+o27Xx9ZJRGVkdVShEtPoqsIx5yALv3lWVU6y+PqYvWPJNWE7ORCQheQkEe0DDA== - dependencies: - "@jsep-plugin/assignment" "^1.2.1" - "@jsep-plugin/regex" "^1.0.3" - jsep "^1.3.8" - jsonschema@1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.4.0.tgz#1afa34c4bc22190d8e42271ec17ac8b3404f87b2" From b8a2c4d6b3872a572eebe0e0e6d5af0448ce309a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:14:25 +0100 Subject: [PATCH 066/192] Lint --- packages/server/src/api/routes/tests/search.spec.ts | 1 - packages/server/src/middleware/zod-validator.ts | 4 ---- 2 files changed, 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5edc4f29ad..5384444067 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -24,7 +24,6 @@ import { JsonFieldSubType, LogicalOperator, RelationshipType, - RequiredKeys, Row, RowSearchParams, SearchFilters, diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts index f990ee0642..0ad346afc3 100644 --- a/packages/server/src/middleware/zod-validator.ts +++ b/packages/server/src/middleware/zod-validator.ts @@ -36,7 +36,3 @@ function validate(schema: AnyZodObject, property: "body" | "params") { export function validateBody(schema: AnyZodObject) { return validate(schema, "body") } - -// export function validateParams(schema: AnyZodObject) { -// return validate(schema, "params") -// } From 0cef937a14c5d40a8f663f7a1baf37aeaeafa217 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:15:19 +0100 Subject: [PATCH 067/192] Update deps --- packages/types/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/types/package.json b/packages/types/package.json index aa67a2e5b4..ae99d31838 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -20,10 +20,10 @@ "@types/redlock": "4.0.7", "rimraf": "3.0.2", "typescript": "5.5.2", - "koa-useragent": "^4.1.0" + "koa-useragent": "^4.1.0", + "zod": "^3.23.8" }, "dependencies": { - "scim-patch": "^0.8.1", - "zod": "^3.23.8" + "scim-patch": "^0.8.1" } } From ab00add0ff77d982bd6cedb279b2f04d8eafa87b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:23:03 +0100 Subject: [PATCH 068/192] Change to nullish --- packages/types/src/api/web/app/rows.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 520983d353..cb213f4a1e 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -78,11 +78,11 @@ const searchRowRequest = z.object({ }) .optional(), paginate: z.boolean().optional(), - bookmark: z.union([z.string(), z.number()]).nullable().optional(), + bookmark: z.union([z.string(), z.number()]).nullish(), limit: z.number().optional(), sort: z.string().optional(), sortOrder: z.nativeEnum(SortOrder).optional(), - sortType: z.nativeEnum(SortType).optional(), + sortType: z.nativeEnum(SortType).nullish(), version: z.string().optional(), disableEscaping: z.boolean().optional(), countRows: z.boolean().optional(), From 792388f1e10d98212fd36223f15ebaacbdbe1b52 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:23:46 +0100 Subject: [PATCH 069/192] Renames --- packages/types/src/api/web/app/{rows.ts => rows/index.ts} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename packages/types/src/api/web/app/{rows.ts => rows/index.ts} (96%) diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows/index.ts similarity index 96% rename from packages/types/src/api/web/app/rows.ts rename to packages/types/src/api/web/app/rows/index.ts index cb213f4a1e..001066d7fb 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows/index.ts @@ -7,13 +7,13 @@ import { RangeOperator, SearchFilterKey, SearchFilters, -} from "../../../sdk" -import { Row } from "../../../documents" +} from "../../../../sdk" +import { Row } from "../../../../documents" import { PaginationResponse, SortOrder, SortType, -} from "../../../api/web/pagination" +} from "../../../../api/web/pagination" import { ReadStream } from "fs" import { z } from "zod" From bb254292d9c2f943eae8ee74f7781f316d52dde3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 15:25:27 +0000 Subject: [PATCH 070/192] migrate bash tests and actually make some useful automations with them --- .../server/src/automations/tests/bash.spec.ts | 152 ++++++++++++++++-- .../tests/utilities/AutomationTestBuilder.ts | 13 ++ 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/packages/server/src/automations/tests/bash.spec.ts b/packages/server/src/automations/tests/bash.spec.ts index 472d1092d6..12ed784268 100644 --- a/packages/server/src/automations/tests/bash.spec.ts +++ b/packages/server/src/automations/tests/bash.spec.ts @@ -1,26 +1,148 @@ -import { getConfig, afterAll as _afterAll, runStep } from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" +import * as automation from "../index" +import * as setup from "./utilities" +import { Table } from "@budibase/types" -describe("test the bash action", () => { - let config = getConfig() +describe("Execute Bash Automations", () => { + let config = setup.getConfig(), + table: Table beforeAll(async () => { + await automation.init() await config.init() - }) - afterAll(_afterAll) - - it("should be able to execute a script", async () => { - let res = await runStep(config, "EXECUTE_BASH", { - code: "echo 'test'", + table = await config.createTable() + await config.createRow({ + name: "test row", + description: "test description", + tableId: table._id!, }) - expect(res.stdout).toEqual("test\n") - expect(res.success).toEqual(true) }) - it("should handle a null value", async () => { - let res = await runStep(config, "EXECUTE_BASH", { - code: null, + afterAll(setup.afterAll) + + it("should use trigger data in bash command and pass output to subsequent steps", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Trigger Data", + config, }) - expect(res.stdout).toEqual( + .appAction({ fields: { command: "hello world" } }) + .bash( + { code: "echo '{{ trigger.fields.command }}'" }, + { stepName: "Echo Command" } + ) + .serverLog( + { text: "Bash output was: {{ steps.[Echo Command].stdout }}" }, + { stepName: "Log Output" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toEqual("hello world\n") + expect(result.steps[1].outputs.message).toContain( + "Bash output was: hello world" + ) + }) + + it("should chain multiple bash commands using previous outputs", async () => { + const result = await createAutomationBuilder({ + name: "Chained Bash Commands", + config, + }) + .appAction({ fields: { filename: "testfile.txt" } }) + .bash( + { code: "echo 'initial content' > {{ trigger.fields.filename }}" }, + { stepName: "Create File" } + ) + .bash( + { code: "cat {{ trigger.fields.filename }} | tr '[a-z]' '[A-Z]'" }, + { stepName: "Transform Content" } + ) + .bash( + { code: "rm {{ trigger.fields.filename }}" }, + { stepName: "Cleanup" } + ) + .run() + + expect(result.steps[1].outputs.stdout).toEqual("INITIAL CONTENT\n") + expect(result.steps[1].outputs.success).toEqual(true) + }) + + it("should integrate bash output with row operations", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Row Operations", + config, + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + }, + { stepName: "Get Row" } + ) + .bash( + { + code: "echo Row data: {{ steps.[Get Row].rows.[0].name }} - {{ steps.[Get Row].rows.[0].description }}", + }, + { stepName: "Process Row Data" } + ) + .serverLog( + { text: "{{ steps.[Process Row Data].stdout }}" }, + { stepName: "Log Result" } + ) + .run() + + expect(result.steps[1].outputs.stdout).toContain( + "Row data: test row - test description" + ) + expect(result.steps[2].outputs.message).toContain( + "Row data: test row - test description" + ) + }) + + it("should handle bash output in conditional logic", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Conditional", + config, + }) + .appAction({ fields: { threshold: "5" } }) + .bash( + { code: "echo $(( {{ trigger.fields.threshold }} + 5 ))" }, + { stepName: "Calculate Value" } + ) + .executeScript( + { + code: ` + const value = parseInt(steps["Calculate Value"].stdout); + return value > 8 ? "high" : "low"; + `, + }, + { stepName: "Check Value" } + ) + .serverLog( + { text: "Value was {{ steps.[Check Value].value }}" }, + { stepName: "Log Result" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toEqual("10\n") + expect(result.steps[1].outputs.value).toEqual("high") + expect(result.steps[2].outputs.message).toContain("Value was high") + }) + + it("should handle null values gracefully", async () => { + const result = await createAutomationBuilder({ + name: "Null Bash Input", + config, + }) + .appAction({ fields: {} }) + .bash( + //@ts-ignore + { code: null }, + { stepName: "Null Command" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toBe( "Budibase bash automation failed: Invalid inputs" ) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 0c091ea00b..7a0d489f80 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -36,6 +36,7 @@ import { FilterStepInputs, ExecuteScriptStepInputs, OpenAIStepInputs, + BashStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -223,6 +224,18 @@ class BaseStepBuilder { ) } + bash( + input: BashStepInputs, + opts?: { stepName?: string; stepId?: string } + ): this { + return this.step( + AutomationActionStepId.EXECUTE_BASH, + BUILTIN_ACTION_DEFINITIONS.EXECUTE_BASH, + input, + opts + ) + } + openai( input: OpenAIStepInputs, opts?: { stepName?: string; stepId?: string } From d4ec07fa8a2100345b5684c6f16fa2d403bf7b94 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Nov 2024 15:30:10 +0000 Subject: [PATCH 071/192] Removing the datasource query API for datasource plus, this isn't used and has no real purpose (internal tables aren't supported) removing it as it exposes very internal types to the API, making it harder to adjust these types. --- .../server/src/api/controllers/datasource.ts | 10 ----- packages/server/src/api/routes/datasource.ts | 9 ---- .../routes/tests/queries/generic-sql.spec.ts | 43 ------------------- .../src/tests/utilities/api/datasource.ts | 10 ----- 4 files changed, 72 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 794e2dfddd..97cf8db299 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -296,16 +296,6 @@ export async function find(ctx: UserCtx) { ctx.body = await sdk.datasources.removeSecretSingle(datasource) } -// dynamic query functionality -export async function query(ctx: UserCtx) { - const queryJson = ctx.request.body - try { - ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) - } catch (err: any) { - ctx.throw(400, err) - } -} - export async function getExternalSchema(ctx: UserCtx) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 755088c56c..22f9a77cc9 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -41,15 +41,6 @@ router ), datasourceController.update ) - .post( - "/api/datasources/query", - authorized( - permissions.PermissionType.TABLE, - permissions.PermissionLevel.READ - ), - datasourceQueryValidator(), - datasourceController.query - ) .post( "/api/datasources/:datasourceId/schema", authorized(permissions.BUILDER), diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 44b21e0350..001f4890fd 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -817,49 +817,6 @@ if (descriptions.length) { }) describe("query through datasource", () => { - it("should be able to query the datasource", async () => { - const datasource = await config.api.datasource.create(rawDatasource) - - const entityId = tableName - await config.api.datasource.update({ - ...datasource, - entities: { - [entityId]: { - name: entityId, - schema: {}, - type: "table", - primary: ["id"], - sourceId: datasource._id!, - sourceType: TableSourceType.EXTERNAL, - }, - }, - }) - - const res = await config.api.datasource.query({ - endpoint: { - datasourceId: datasource._id!, - operation: Operation.READ, - entityId, - }, - resource: { - fields: ["id", "name"], - }, - filters: { - string: { - name: "two", - }, - }, - }) - expect(res).toHaveLength(1) - expect(res[0]).toEqual({ - id: 2, - name: "two", - // the use of table.* introduces the possibility of nulls being returned - birthday: null, - number: null, - }) - }) - // this parameter really only impacts SQL queries describe("confirm nullDefaultSupport", () => { let queryParams: Partial diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 67484a688a..9be7be998d 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -66,16 +66,6 @@ export class DatasourceAPI extends TestAPI { return await this._get(`/api/datasources`, { expectations }) } - query = async ( - query: Omit & Partial>, - expectations?: Expectations - ) => { - return await this._post(`/api/datasources/query`, { - body: query, - expectations, - }) - } - fetchSchema = async ( { datasourceId, From e3e82f2fb84ddea121634a6e8ad28a15a467bc19 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:33:31 +0100 Subject: [PATCH 072/192] Move types --- packages/types/src/api/web/app/rows/index.ts | 104 +----------------- packages/types/src/api/web/app/rows/search.ts | 100 +++++++++++++++++ 2 files changed, 104 insertions(+), 100 deletions(-) create mode 100644 packages/types/src/api/web/app/rows/search.ts diff --git a/packages/types/src/api/web/app/rows/index.ts b/packages/types/src/api/web/app/rows/index.ts index 001066d7fb..2642a8b04e 100644 --- a/packages/types/src/api/web/app/rows/index.ts +++ b/packages/types/src/api/web/app/rows/index.ts @@ -1,21 +1,9 @@ -import { - ArrayOperator, - BasicOperator, - EmptyFilterOption, - InternalSearchFilterOperator, - LogicalOperator, - RangeOperator, - SearchFilterKey, - SearchFilters, -} from "../../../../sdk" +import { SearchFilters } from "../../../../sdk" import { Row } from "../../../../documents" -import { - PaginationResponse, - SortOrder, - SortType, -} from "../../../../api/web/pagination" +import { SortOrder } from "../../../../api/web/pagination" import { ReadStream } from "fs" -import { z } from "zod" + +export * from "./search" export interface SaveRowRequest extends Row {} @@ -27,90 +15,6 @@ export interface PatchRowRequest extends Row { export interface PatchRowResponse extends Row {} -const fieldKey = z - .string() - .refine(s => s !== InternalSearchFilterOperator.COMPLEX_ID_OPERATOR, { - message: `Key '${InternalSearchFilterOperator.COMPLEX_ID_OPERATOR}' is not allowed`, - }) - -const stringBasicFilter = z.record(fieldKey, z.string()) -const basicFilter = z.record(fieldKey, z.any()) -const arrayFilter = z.record(fieldKey, z.union([z.any().array(), z.string()])) -const logicFilter = z.lazy(() => - z.object({ - conditions: z.array(z.object(queryFilterValidation)), - }) -) - -const stringOrNumber = z.union([z.string(), z.number()]) - -const queryFilterValidation: Record = { - [BasicOperator.STRING]: stringBasicFilter.optional(), - [BasicOperator.FUZZY]: stringBasicFilter.optional(), - [RangeOperator.RANGE]: z - .record( - fieldKey, - z.union([ - z.object({ high: stringOrNumber, low: stringOrNumber }), - z.object({ high: stringOrNumber }), - z.object({ low: stringOrNumber }), - ]) - ) - .optional(), - [BasicOperator.EQUAL]: basicFilter.optional(), - [BasicOperator.NOT_EQUAL]: basicFilter.optional(), - [BasicOperator.EMPTY]: basicFilter.optional(), - [BasicOperator.NOT_EMPTY]: basicFilter.optional(), - [ArrayOperator.ONE_OF]: arrayFilter.optional(), - [ArrayOperator.CONTAINS]: arrayFilter.optional(), - [ArrayOperator.NOT_CONTAINS]: arrayFilter.optional(), - [ArrayOperator.CONTAINS_ANY]: arrayFilter.optional(), - [LogicalOperator.AND]: logicFilter.optional(), - [LogicalOperator.OR]: logicFilter.optional(), -} - -const searchRowRequest = z.object({ - query: z - .object({ - allOr: z.boolean().optional(), - onEmptyFilter: z.nativeEnum(EmptyFilterOption).optional(), - ...queryFilterValidation, - }) - .optional(), - paginate: z.boolean().optional(), - bookmark: z.union([z.string(), z.number()]).nullish(), - limit: z.number().optional(), - sort: z.string().optional(), - sortOrder: z.nativeEnum(SortOrder).optional(), - sortType: z.nativeEnum(SortType).nullish(), - version: z.string().optional(), - disableEscaping: z.boolean().optional(), - countRows: z.boolean().optional(), -}) -export const searchRowRequestValidator = searchRowRequest -export type SearchRowRequest = z.infer - -export interface SearchViewRowRequest - extends Pick< - SearchRowRequest, - | "sort" - | "sortOrder" - | "sortType" - | "limit" - | "bookmark" - | "paginate" - | "query" - | "countRows" - > {} - -export interface SearchRowResponse { - rows: any[] -} - -export interface PaginatedSearchRowResponse - extends SearchRowResponse, - PaginationResponse {} - export interface ExportRowsRequest { rows?: string[] columns?: string[] diff --git a/packages/types/src/api/web/app/rows/search.ts b/packages/types/src/api/web/app/rows/search.ts new file mode 100644 index 0000000000..b74e490677 --- /dev/null +++ b/packages/types/src/api/web/app/rows/search.ts @@ -0,0 +1,100 @@ +import { + ArrayOperator, + BasicOperator, + EmptyFilterOption, + InternalSearchFilterOperator, + LogicalOperator, + RangeOperator, + SearchFilterKey, +} from "../../../../sdk" +import { Row } from "../../../../documents" +import { + PaginationResponse, + SortOrder, + SortType, +} from "../../../../api/web/pagination" +import { z } from "zod" + +const fieldKey = z + .string() + .refine(s => s !== InternalSearchFilterOperator.COMPLEX_ID_OPERATOR, { + message: `Key '${InternalSearchFilterOperator.COMPLEX_ID_OPERATOR}' is not allowed`, + }) + +const stringBasicFilter = z.record(fieldKey, z.string()) +const basicFilter = z.record(fieldKey, z.any()) +const arrayFilter = z.record(fieldKey, z.union([z.any().array(), z.string()])) +const logicFilter = z.lazy(() => + z.object({ + conditions: z.array(z.object(queryFilterValidation)), + }) +) + +const stringOrNumber = z.union([z.string(), z.number()]) + +const queryFilterValidation: Record = { + [BasicOperator.STRING]: stringBasicFilter.optional(), + [BasicOperator.FUZZY]: stringBasicFilter.optional(), + [RangeOperator.RANGE]: z + .record( + fieldKey, + z.union([ + z.object({ high: stringOrNumber, low: stringOrNumber }), + z.object({ high: stringOrNumber }), + z.object({ low: stringOrNumber }), + ]) + ) + .optional(), + [BasicOperator.EQUAL]: basicFilter.optional(), + [BasicOperator.NOT_EQUAL]: basicFilter.optional(), + [BasicOperator.EMPTY]: basicFilter.optional(), + [BasicOperator.NOT_EMPTY]: basicFilter.optional(), + [ArrayOperator.ONE_OF]: arrayFilter.optional(), + [ArrayOperator.CONTAINS]: arrayFilter.optional(), + [ArrayOperator.NOT_CONTAINS]: arrayFilter.optional(), + [ArrayOperator.CONTAINS_ANY]: arrayFilter.optional(), + [LogicalOperator.AND]: logicFilter.optional(), + [LogicalOperator.OR]: logicFilter.optional(), +} + +const searchRowRequest = z.object({ + query: z + .object({ + allOr: z.boolean().optional(), + onEmptyFilter: z.nativeEnum(EmptyFilterOption).optional(), + ...queryFilterValidation, + }) + .optional(), + paginate: z.boolean().optional(), + bookmark: z.union([z.string(), z.number()]).nullish(), + limit: z.number().optional(), + sort: z.string().optional(), + sortOrder: z.nativeEnum(SortOrder).optional(), + sortType: z.nativeEnum(SortType).nullish(), + version: z.string().optional(), + disableEscaping: z.boolean().optional(), + countRows: z.boolean().optional(), +}) + +export const searchRowRequestValidator = searchRowRequest + +export type SearchRowRequest = z.infer +export type SearchViewRowRequest = Pick< + SearchRowRequest, + | "sort" + | "sortOrder" + | "sortType" + | "limit" + | "bookmark" + | "paginate" + | "query" + | "countRows" +> + +export interface SearchRowResponse { + rows: Row[] +} + +export interface PaginatedSearchRowResponse + extends SearchRowResponse, + PaginationResponse {} From 7ed45f71d273b1d12f6c377181a1c40d850a5d22 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:35:24 +0100 Subject: [PATCH 073/192] Fix type tests --- .../src/api/routes/tests/search.spec.ts | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5384444067..8ba165ded0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -24,6 +24,7 @@ import { JsonFieldSubType, LogicalOperator, RelationshipType, + RequiredKeys, Row, RowSearchParams, SearchFilters, @@ -208,9 +209,25 @@ if (descriptions.length) { private async performSearch(): Promise> { if (isInMemory) { - return dataFilters.search(_.cloneDeep(rows), { - ...this.query, - }) + const inMemoryQuery: RequiredKeys< + Omit + > = { + sort: this.query.sort, + query: { ...this.query.query }, + paginate: this.query.paginate, + bookmark: this.query.bookmark ?? undefined, + limit: this.query.limit, + sortOrder: this.query.sortOrder, + sortType: this.query.sortType ?? undefined, + version: this.query.version, + disableEscaping: this.query.disableEscaping, + countRows: this.query.countRows, + viewId: undefined, + fields: undefined, + indexer: undefined, + rows: undefined, + } + return dataFilters.search(_.cloneDeep(rows), inMemoryQuery) } else { return config.api.row.search(tableOrViewId, this.query) } From 22d6b951013304cb003dbdc2684c902645f604e9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 15:39:16 +0000 Subject: [PATCH 074/192] Put errors back how they were. --- packages/server/src/api/controllers/table/external.ts | 2 +- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 5e77dc1dc6..6f09bf4a61 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -54,7 +54,7 @@ export async function updateTable( return table } catch (err: any) { if (err instanceof Error) { - throw err + ctx.throw(400, err.message) } else { ctx.throw(err.status || 500, err?.message || err) } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index ba5614edca..9bfa0ecd27 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -485,7 +485,6 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - // throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) - throw err + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } From 0501980e45c25ef0f13c3240fb6168abcec75532 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 16:44:25 +0100 Subject: [PATCH 075/192] Fix types --- packages/server/src/api/controllers/row/index.ts | 2 +- packages/server/src/api/controllers/row/views.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index fbaf9f8462..2dc9381183 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -260,7 +260,7 @@ export async function search(ctx: Ctx) { limit: searchRequest.limit, sort: searchRequest.sort, sortOrder: searchRequest.sortOrder, - sortType: searchRequest.sortType, + sortType: searchRequest.sortType ?? undefined, countRows: searchRequest.countRows, version: searchRequest.version, disableEscaping: searchRequest.disableEscaping, diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index eb010fee98..0655a3b38f 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -57,7 +57,7 @@ function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { return { sort: request.sort, sortOrder: request.sortOrder, - sortType: request.sortType, + sortType: request.sortType ?? undefined, } } if (view.sort) { From b17c6ab43add99011aecef589658c4c01d21c1af Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:01:28 +0000 Subject: [PATCH 076/192] Cleanup following bigint fixes. --- packages/backend-core/src/sql/sql.ts | 56 ++++++------------- packages/backend-core/src/sql/sqlTable.ts | 15 +++-- packages/pro | 2 +- .../server/src/integrations/googlesheets.ts | 4 +- .../src/sdk/app/rows/search/internal/sqs.ts | 2 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 8 +-- packages/server/src/sdk/app/rows/utils.ts | 2 + packages/types/src/sdk/search.ts | 8 +-- 8 files changed, 36 insertions(+), 61 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index d251ff672d..5f462ee144 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -284,13 +284,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource } = this.query + const { table, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = this.getTableName(endpoint.entityId) + const alias = this.getTableName(table) const schema = this.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw("??", [`${alias}.*`])] @@ -496,7 +496,7 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const { relationships, endpoint, tableAliases: aliases, table } = this.query + const { relationships, schema, tableAliases: aliases, table } = this.query const fromAlias = aliases?.[table.name] || table.name const matches = (value: string) => filterKey.match(new RegExp(`^${value}\\.`)) @@ -537,7 +537,7 @@ class InternalBuilder { aliases?.[manyToMany.through] || relationship.through let throughTable = this.tableNameWithSchema(manyToMany.through, { alias: throughAlias, - schema: endpoint.schema, + schema, }) subQuery = subQuery // add a join through the junction table @@ -1010,28 +1010,10 @@ class InternalBuilder { return isSqs(this.table) } - getTableName(tableOrName?: Table | string): string { - let table: Table - if (typeof tableOrName === "string") { - const name = tableOrName - if (this.query.table?.name === name) { - table = this.query.table - } else if (this.query.table.name === name) { - table = this.query.table - } else if (!this.query.tables[name]) { - // This can legitimately happen in custom queries, where the user is - // querying against a table that may not have been imported into - // Budibase. - return name - } else { - table = this.query.tables[name] - } - } else if (tableOrName) { - table = tableOrName - } else { + getTableName(table?: Table): string { + if (!table) { table = this.table } - let name = table.name if (isSqs(table) && table._id) { // SQS uses the table ID rather than the table name @@ -1242,7 +1224,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint, tables } = this.query + const { resource, tableAliases: aliases, schema, tables } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -1266,7 +1248,7 @@ class InternalBuilder { throughAlias = (throughTable && aliases?.[throughTable]) || throughTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, - schema: endpoint.schema, + schema, }) const requiredFields = [ ...(relatedTable?.primary || []), @@ -1310,7 +1292,7 @@ class InternalBuilder { if (isManyToMany) { let throughTableWithSchema = this.tableNameWithSchema(throughTable, { alias: throughAlias, - schema: endpoint.schema, + schema, }) subQuery = subQuery.join(throughTableWithSchema, function () { this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) @@ -1401,8 +1383,7 @@ class InternalBuilder { toPrimary?: string }[] ): Knex.QueryBuilder { - const { tableAliases: aliases, endpoint } = this.query - const schema = endpoint.schema + const { tableAliases: aliases, schema } = this.query const toTable = tables.to, fromTable = tables.from, throughTable = tables.through @@ -1462,7 +1443,7 @@ class InternalBuilder { return this.knex( this.tableNameWithSchema(this.query.table.name, { alias, - schema: this.query.endpoint.schema, + schema: this.query.schema, }) ) } @@ -1556,9 +1537,8 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships, table } = this.query + let { operation, filters, paginate, relationships, table } = this.query const { limits } = opts - const counting = endpoint.operation === Operation.COUNT // start building the query let query = this.qualifiedKnex() @@ -1578,7 +1558,7 @@ class InternalBuilder { foundLimit = paginate.limit } // counting should not sort, limit or offset - if (!counting) { + if (operation !== Operation.COUNT) { // add the found limit if supplied if (foundLimit != null) { query = query.limit(foundLimit) @@ -1590,7 +1570,7 @@ class InternalBuilder { } const aggregations = this.query.resource?.aggregations || [] - if (counting) { + if (operation === Operation.COUNT) { query = this.addDistinctCount(query) } else if (aggregations.length > 0) { query = this.addAggregations(query, aggregations) @@ -1599,7 +1579,7 @@ class InternalBuilder { } // have to add after as well (this breaks MS-SQL) - if (!counting) { + if (operation !== Operation.COUNT) { query = this.addSorting(query) } @@ -1738,13 +1718,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return {} } const input = this._query({ + operation: Operation.READ, datasource: json.datasource, + schema: json.schema, table: json.table, tables: json.tables, - endpoint: { - ...json.endpoint, - operation: Operation.READ, - }, resource: { fields: [] }, filters: json.extra?.idFilter, paginate: { limit: 1 }, diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 5b76e155f9..8e15d3d4ef 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -239,14 +239,13 @@ class SqlTableQueryBuilder { * @return the operation that was found in the JSON. */ _operation(json: EnrichedQueryJson): Operation { - return json.endpoint.operation + return json.operation } _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] { let client = knex({ client: this.sqlClient }).schema - let schemaName = json?.endpoint?.schema - if (schemaName) { - client = client.withSchema(schemaName) + if (json?.schema) { + client = client.withSchema(json.schema) } let query: Knex.SchemaBuilder @@ -268,8 +267,8 @@ class SqlTableQueryBuilder { // renameColumn does not work for MySQL, so return a raw query if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) { const updatedColumn = json.meta.renamed.updated - const tableName = schemaName - ? `\`${schemaName}\`.\`${json.table.name}\`` + const tableName = json?.schema + ? `\`${json.schema}\`.\`${json.table.name}\`` : `\`${json.table.name}\`` return { sql: `alter table ${tableName} rename column \`${json.meta.renamed.old}\` to \`${updatedColumn}\`;`, @@ -290,8 +289,8 @@ class SqlTableQueryBuilder { if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) { const oldColumn = json.meta.renamed.old const updatedColumn = json.meta.renamed.updated - const tableName = schemaName - ? `${schemaName}.${json.table.name}` + const tableName = json?.schema + ? `${json.schema}.${json.table.name}` : `${json.table.name}` const sql = getNativeSql(query) if (Array.isArray(sql)) { diff --git a/packages/pro b/packages/pro index 6f38253253..49265a7430 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 6f38253253ee364aea636add990083ca5cda3bde +Subproject commit 49265a7430c590968ba65890a91dd398e5312941 diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6492a651e1..0eeb38c78f 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -383,7 +383,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { async query(json: EnrichedQueryJson): Promise { const sheet = json.table.name - switch (json.endpoint.operation) { + switch (json.operation) { case Operation.CREATE: return this.create({ sheet, row: json.body as Row }) case Operation.BULK_CREATE: @@ -426,7 +426,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return this.deleteTable(json?.table?.name) default: throw new Error( - `GSheets integration does not support "${json.endpoint.operation}".` + `GSheets integration does not support "${json.operation}".` ) } } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 9bfa0ecd27..19380bde43 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -231,7 +231,7 @@ async function runSqlQuery( tables.map(table => table._id!).concat(relationshipJunctionTableIds) ) if (opts?.countTotalRows) { - json.endpoint.operation = Operation.COUNT + json.operation = Operation.COUNT } const processSQLQuery = async (json: EnrichedQueryJson) => { const query = builder._query(json, { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 62e0db9aad..f3c3edc053 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -3,7 +3,6 @@ import { DatasourcePlusQueryResponse, EnrichedQueryJson, Operation, - QueryJson, Row, SearchFilters, SqlClient, @@ -69,13 +68,12 @@ export default class AliasTables { this.charSeq = new CharSequence() } - isAliasingEnabled(json: QueryJson, datasource?: Datasource) { - const operation = json.endpoint.operation + isAliasingEnabled(json: EnrichedQueryJson, datasource?: Datasource) { const fieldLength = json.resource?.fields?.length if ( !fieldLength || fieldLength <= 0 || - DISABLED_OPERATIONS.includes(operation) + DISABLED_OPERATIONS.includes(json.operation) ) { return false } @@ -85,7 +83,7 @@ export default class AliasTables { } try { const sqlClient = getSQLClient(datasource) - const isWrite = WRITE_OPERATIONS.includes(operation) + const isWrite = WRITE_OPERATIONS.includes(json.operation) const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient) if (isWrite && isDisabledClient) { return false diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 977cdb9bd2..50170dd8bd 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -118,9 +118,11 @@ export async function enrichQueryJson( } return { + operation: json.endpoint.operation, table, tables, datasource, + schema: json.endpoint.schema, ...json, } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index e25390bdd0..fdc2fafe57 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -184,14 +184,12 @@ export interface QueryJson { tableAliases?: Record } -export interface EnrichedQueryJson extends QueryJson { +export interface EnrichedQueryJson extends Omit { + operation: Operation table: Table tables: Record datasource?: Datasource -} - -export interface QueryJsonRequest extends Omit { - endpoint: QueryJson["endpoint"] & { datasourceId: string; entityId: string } + schema?: string } export interface QueryOptions { From f6ad53f442c8a4c93f37d37339d959c3496f6f51 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:17:09 +0000 Subject: [PATCH 077/192] Follow-up PR to bigint fixes. --- .../server/src/api/controllers/datasource.ts | 2 -- .../server/src/integrations/base/query.ts | 34 +++++++++++++------ .../src/integrations/microsoftSqlServer.ts | 4 +-- packages/server/src/integrations/oracle.ts | 6 +--- .../src/integrations/tests/sqlAlias.spec.ts | 2 +- .../src/tests/utilities/api/datasource.ts | 1 - 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 42d657cf6c..97cf8db299 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,13 +23,11 @@ import { Table, RowValue, DynamicVariable, - QueryJsonRequest, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" -import { makeExternalQuery } from "../../integrations/base/query" export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 6555a5de51..2cbbbecc8d 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -1,27 +1,39 @@ -import { DatasourcePlusQueryResponse, QueryJson } from "@budibase/types" +import { + DatasourcePlusQueryResponse, + EnrichedQueryJson, + QueryJson, +} from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" import { enrichQueryJson } from "../../sdk/app/rows/utils" +function isEnriched( + json: QueryJson | EnrichedQueryJson +): json is EnrichedQueryJson { + return "datasource" in json +} + export async function makeExternalQuery( - json: QueryJson + json: QueryJson | EnrichedQueryJson ): Promise { - const enrichedJson = await enrichQueryJson(json) - if (!enrichedJson.datasource) { + if (!isEnriched(json)) { + json = await enrichQueryJson(json) + if (json.datasource) { + json.datasource = await sdk.datasources.enrich(json.datasource) + } + } + + if (!json.datasource) { throw new Error("No datasource provided for external query") } - enrichedJson.datasource = await sdk.datasources.enrich( - enrichedJson.datasource - ) - - const Integration = await getIntegration(enrichedJson.datasource.source) + const Integration = await getIntegration(json.datasource.source) // query is the opinionated function if (!Integration.prototype.query) { throw "Datasource does not support query." } - const integration = new Integration(enrichedJson.datasource.config) - return integration.query(enrichedJson) + const integration = new Integration(json.datasource.config) + return integration.query(json) } diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index a7a0b53312..2fac306590 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -509,8 +509,8 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { async query(json: EnrichedQueryJson): Promise { const schema = this.config.schema await this.connect() - if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) { - json.endpoint.schema = schema + if (schema && schema !== DEFAULT_SCHEMA) { + json.schema = schema } const operation = this._operation(json) const queryFn = (query: any, op: string) => this.internalQuery(query, op) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index ded21487f6..d8708d9053 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -572,11 +572,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { return response.rows as Row[] } else { // get the last row that was updated - if ( - response.lastRowid && - json.endpoint?.entityId && - operation !== Operation.DELETE - ) { + if (response.lastRowid && operation !== Operation.DELETE) { const lastRow = await this.internalQuery({ sql: `SELECT * FROM "${json.table.name}" WHERE ROWID = '${response.lastRowid}'`, }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 4de84f689e..0f43e41755 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -269,7 +269,7 @@ describe("Captures of real examples", () => { fields: string[] = ["a"] ): EnrichedQueryJson { return { - endpoint: { datasourceId: "", entityId: "", operation: op }, + operation: op, resource: { fields, }, diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index ebd1b9f7c7..0bc365661e 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -4,7 +4,6 @@ import { Datasource, FetchDatasourceInfoResponse, FieldType, - QueryJsonRequest, RelationshipType, UpdateDatasourceRequest, UpdateDatasourceResponse, From 69aa5efcd1aad9227bb7a3e8790add668516af85 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:23:18 +0000 Subject: [PATCH 078/192] Fix sqlAlias.spec.ts --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- .../tests/sqlQueryJson/basicFetchWithRelationships.json | 6 +----- .../src/integrations/tests/sqlQueryJson/createSimple.json | 6 +----- .../tests/sqlQueryJson/createWithRelationships.json | 6 +----- .../src/integrations/tests/sqlQueryJson/deleteSimple.json | 6 +----- .../integrations/tests/sqlQueryJson/enrichRelationship.json | 6 +----- .../integrations/tests/sqlQueryJson/fetchManyToMany.json | 6 +----- .../tests/sqlQueryJson/filterByRelationship.json | 6 +----- .../tests/sqlQueryJson/manyRelationshipFilters.json | 6 +----- .../integrations/tests/sqlQueryJson/updateRelationship.json | 6 +----- .../src/integrations/tests/sqlQueryJson/updateSimple.json | 6 +----- 11 files changed, 11 insertions(+), 51 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5384444067..6a56ec37c6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe("primaryDisplay", () => { + describe.only("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index 27d6efec29..c61d66d684 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "persons", - "operation": "READ" - }, + "operation": "READ", "resource": { "fields": [ "a.year", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 474be32cf2..ac487e4e90 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", - "entityId": "people", - "operation": "CREATE" - }, + "operation": "CREATE", "resource": { "fields": ["a.name", "a.age"] }, diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 060197dd01..571fbcf4ac 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "persons", - "operation": "CREATE" - }, + "operation": "CREATE", "resource": { "fields": [ "a.year", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index d9fc34165f..4a73fc2e8b 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "compositetable", - "operation": "DELETE" - }, + "operation": "DELETE", "resource": { "fields": ["a.keyparttwo", "a.keypartone", "a.name"] }, diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index f24b46859f..3972982168 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "entityId": "tasks", - "operation": "READ" - }, + "operation": "READ", "resource": { "fields": [ "a.executorid", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index b746e05c5e..55ddde917f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "entityId": "products", - "operation": "READ" - }, + "operation": "READ", "resource": { "fields": [ "a.productname", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index 6a9717f5dd..6930c900a2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "products", - "operation": "READ" - }, + "operation": "READ", "resource": { "fields": [ "a.productname", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index 23f56796a9..5f5ebb95f3 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "entityId": "tasks", - "operation": "READ" - }, + "operation": "READ", "resource": { "fields": [ "a.executorid", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 8171e22c64..59a978904f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "persons", - "operation": "UPDATE" - }, + "operation": "UPDATE", "resource": { "fields": [ "a.year", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 8171e22c64..59a978904f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -1,9 +1,5 @@ { - "endpoint": { - "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "entityId": "persons", - "operation": "UPDATE" - }, + "operation": "UPDATE", "resource": { "fields": [ "a.year", From 29a6a6683ba9916c5326f47e7462c74a4e88c63f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:34:52 +0000 Subject: [PATCH 079/192] Fix link. --- packages/server/src/api/routes/datasource.ts | 5 +---- .../src/api/routes/tests/queries/generic-sql.spec.ts | 8 +------- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 22f9a77cc9..9ec792632a 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -2,10 +2,7 @@ import Router from "@koa/router" import * as datasourceController from "../controllers/datasource" import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" -import { - datasourceValidator, - datasourceQueryValidator, -} from "./utils/validators" +import { datasourceValidator } from "./utils/validators" const router: Router = new Router() diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 001f4890fd..e7ddc0df22 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -1,10 +1,4 @@ -import { - Datasource, - Operation, - Query, - QueryPreview, - TableSourceType, -} from "@budibase/types" +import { Datasource, Query, QueryPreview } from "@budibase/types" import { DatabaseName, datasourceDescribe, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6a56ec37c6..5384444067 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe.only("primaryDisplay", () => { + describe("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { From 00dc5d60a2df119dbd834a390bc7fb71198a5b69 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:35:36 +0000 Subject: [PATCH 080/192] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 49265a7430..aa6517bfbc 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 49265a7430c590968ba65890a91dd398e5312941 +Subproject commit aa6517bfbc463b3aac4ae53245dcce3d8bb130e4 From a162d8d9f584ff5d255e2799c8e62bd95f947bd2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Nov 2024 16:43:21 +0000 Subject: [PATCH 081/192] Linting. --- packages/server/src/api/routes/datasource.ts | 5 +---- .../src/api/routes/tests/queries/generic-sql.spec.ts | 8 +------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 22f9a77cc9..9ec792632a 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -2,10 +2,7 @@ import Router from "@koa/router" import * as datasourceController from "../controllers/datasource" import authorized from "../../middleware/authorized" import { permissions } from "@budibase/backend-core" -import { - datasourceValidator, - datasourceQueryValidator, -} from "./utils/validators" +import { datasourceValidator } from "./utils/validators" const router: Router = new Router() diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 001f4890fd..e7ddc0df22 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -1,10 +1,4 @@ -import { - Datasource, - Operation, - Query, - QueryPreview, - TableSourceType, -} from "@budibase/types" +import { Datasource, Query, QueryPreview } from "@budibase/types" import { DatabaseName, datasourceDescribe, From 69106306c4f50de1aece5fc74383f4b9716d40c6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Nov 2024 16:43:47 +0000 Subject: [PATCH 082/192] Removing unused validator. --- .../server/src/api/routes/utils/validators.ts | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 3bee4f88ce..30862cfc68 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -231,30 +231,6 @@ export function externalSearchValidator() { ) } -export function datasourceQueryValidator() { - return auth.joiValidator.body( - Joi.object({ - endpoint: Joi.object({ - datasourceId: Joi.string().required(), - operation: Joi.string() - .required() - .valid(...Object.values(DataSourceOperation)), - entityId: Joi.string().required(), - }).required(), - resource: Joi.object({ - fields: Joi.array().items(Joi.string()).optional(), - }).optional(), - body: Joi.object().optional(), - sort: Joi.object().optional(), - filters: filterObject().optional(), - paginate: Joi.object({ - page: Joi.string().alphanum().optional(), - limit: Joi.number().optional(), - }).optional(), - }) - ) -} - export function webhookValidator() { return auth.joiValidator.body( Joi.object({ From 8b653d8dd9184a696633ae4be8844188ff06b451 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 28 Nov 2024 16:45:43 +0000 Subject: [PATCH 083/192] Update pro ref (#15088) --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 6f38253253..e2252498dd 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 6f38253253ee364aea636add990083ca5cda3bde +Subproject commit e2252498ddfade3c2592b1ec78f7bee4e3cf0d2f From f91d442ad46169b2f97856263dfc98f4a591eee3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 16:48:28 +0000 Subject: [PATCH 084/192] Remove exited containers when running tests. --- .../core/utilities/testContainerUtils.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 71d7fa32db..727b290183 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -25,7 +25,7 @@ function getTestcontainers(): ContainerInfo[] { // We use --format json to make sure the output is nice and machine-readable, // and we use --no-trunc so that the command returns full container IDs so we // can filter on them correctly. - return execSync("docker ps --format json --no-trunc") + return execSync("docker ps --all --format json --no-trunc") .toString() .split("\n") .filter(x => x.length > 0) @@ -37,6 +37,10 @@ function getTestcontainers(): ContainerInfo[] { ) } +function removeContainer(container: ContainerInfo) { + execSync(`docker rm ${container.ID}`) +} + export function getContainerByImage(image: string) { const containers = getTestcontainers().filter(x => x.Image.startsWith(image)) if (containers.length > 1) { @@ -49,6 +53,10 @@ export function getContainerByImage(image: string) { return containers[0] } +function getContainerByName(name: string) { + return getTestcontainers().find(x => x.Names === name) +} + export function getContainerById(id: string) { return getTestcontainers().find(x => x.ID === id) } @@ -116,6 +124,16 @@ export async function startContainer(container: GenericContainer) { key = imageName.split("@")[0] } key = key.replace(/\//g, "-").replace(/:/g, "-") + const name = `${key}_testcontainer` + + // If a container has died it hangs around and future attempts to start a + // container with the same name will fail. What we do here is if we find a + // matching container and it has exited, we remove it before carrying on. This + // removes the need to do this removal manually. + const existingContainer = getContainerByName(name) + if (existingContainer?.State === "exited") { + removeContainer(existingContainer) + } container = container .withReuse() From 13918c5789fa6e1195cda7376e2c6bfda885fe45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Nov 2024 16:49:35 +0000 Subject: [PATCH 085/192] More linting. --- packages/server/src/api/routes/utils/validators.ts | 1 - packages/server/src/constants/index.ts | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 30862cfc68..a8fa12ec3d 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -1,5 +1,4 @@ import { auth, permissions } from "@budibase/backend-core" -import { DataSourceOperation } from "../../../constants" import { AutomationActionStepId, AutomationStep, diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index bac838b53e..604a81cd9f 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -45,17 +45,6 @@ export enum AuthTypes { EXTERNAL = "external", } -export enum DataSourceOperation { - CREATE = "CREATE", - READ = "READ", - UPDATE = "UPDATE", - DELETE = "DELETE", - BULK_CREATE = "BULK_CREATE", - CREATE_TABLE = "CREATE_TABLE", - UPDATE_TABLE = "UPDATE_TABLE", - DELETE_TABLE = "DELETE_TABLE", -} - export enum DatasourceAuthTypes { GOOGLE = "google", } From 3cf2dbe37ce40c0cab4b9f3ffde523bbd6802db1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 17:16:23 +0000 Subject: [PATCH 086/192] Make sure we're always using the current docker context. --- globalSetup.ts | 28 +++++++++++++++++++ .../core/utilities/testContainerUtils.ts | 27 ++++++++++++++++++ .../src/api/routes/tests/search.spec.ts | 2 +- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/globalSetup.ts b/globalSetup.ts index 07a0cec5e2..9ebda5626d 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -6,6 +6,26 @@ import { import { ContainerInfo } from "dockerode" import path from "path" import lockfile from "proper-lockfile" +import { execSync } from "child_process" + +interface DockerContext { + Name: string + Description: string + DockerEndpoint: string + ContextType: string + Error: string +} + +function getCurrentDockerContext(): DockerContext { + const out = execSync("docker context ls --format json") + for (const line of out.toString().split("\n")) { + const parsed = JSON.parse(line) + if (parsed.Current) { + return parsed as DockerContext + } + } + throw new Error("No current Docker context") +} async function getBudibaseContainers() { const client = await getContainerRuntimeClient() @@ -27,6 +47,14 @@ async function killContainers(containers: ContainerInfo[]) { } export default async function setup() { + // For whatever reason, testcontainers doesn't always use the correct current + // docker context. This bit of code forces the issue by finding the current + // context and setting it as the DOCKER_HOST environment + if (!process.env.DOCKER_HOST) { + const dockerContext = getCurrentDockerContext() + process.env.DOCKER_HOST = dockerContext.DockerEndpoint + } + const lockPath = path.resolve(__dirname, "globalSetup.ts") // If you run multiple tests at the same time, it's possible for the CouchDB // shared container to get started multiple times despite having an diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 727b290183..339656cf09 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -78,7 +78,34 @@ export function getExposedV4Port(container: ContainerInfo, port: number) { return getExposedV4Ports(container).find(x => x.container === port)?.host } +interface DockerContext { + Name: string + Description: string + DockerEndpoint: string + ContextType: string + Error: string +} + +function getCurrentDockerContext(): DockerContext { + const out = execSync("docker context ls --format json") + for (const line of out.toString().split("\n")) { + const parsed = JSON.parse(line) + if (parsed.Current) { + return parsed as DockerContext + } + } + throw new Error("No current Docker context") +} + export function setupEnv(...envs: any[]) { + // For whatever reason, testcontainers doesn't always use the correct current + // docker context. This bit of code forces the issue by finding the current + // context and setting it as the DOCKER_HOST environment + if (!process.env.DOCKER_HOST) { + const dockerContext = getCurrentDockerContext() + process.env.DOCKER_HOST = dockerContext.DockerEndpoint + } + // We start couchdb in globalSetup.ts, in the root of the monorepo, so it // should be relatively safe to look for it by its image name. const couch = getContainerByImage("budibase/couchdb") diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5384444067..6a56ec37c6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe("primaryDisplay", () => { + describe.only("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { From bcd8bcdc1c9e1446148d9dc76e024e3ae3508adf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 28 Nov 2024 17:17:29 +0000 Subject: [PATCH 087/192] Fixing issue detected by test. --- packages/server/src/api/controllers/row/staticFormula.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index ff2381e61c..4464b7f44a 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -179,6 +179,7 @@ export async function finaliseRow( throw new Error(`Unable to retrieve row ${row._id} after saving.`) } + delete enrichedRow._rev enrichedRow = mergeRows(retrieved, enrichedRow) enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, From 3bff9f2e551005a98a6448d4faeacde1509fa425 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 17:23:16 +0000 Subject: [PATCH 088/192] Unfocus test. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6a56ec37c6..5384444067 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3329,7 +3329,7 @@ if (descriptions.length) { }) isSql && - describe.only("primaryDisplay", () => { + describe("primaryDisplay", () => { beforeAll(async () => { let toRelateTableId = await createTable({ name: { From 4f915f45225b1d30ddaba1924c576a7846e492c4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 28 Nov 2024 17:37:40 +0000 Subject: [PATCH 089/192] Update pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index abdfb720c5..d9245f3d6d 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit abdfb720c5583256e0334330ccbea1abcbe3d434 +Subproject commit d9245f3d6d0b41ec2e6b3406b791f9e7448882cb From 9982b5d9076e3a837ec155d4bb94e74712151abf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Nov 2024 10:57:15 +0100 Subject: [PATCH 090/192] Change bbui dependencies from rollup to vite --- packages/bbui/package.json | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index aeb7418526..02cb81c2c4 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -14,17 +14,12 @@ "./spectrum-icons-vite.js": "./src/spectrum-icons-vite.js" }, "scripts": { - "build": "rollup -c" + "build": "vite build" }, "devDependencies": { - "@rollup/plugin-commonjs": "^16.0.0", - "@rollup/plugin-json": "^4.1.0", - "@rollup/plugin-node-resolve": "^11.2.1", + "@sveltejs/vite-plugin-svelte": "1.4.0", "postcss": "^8.2.9", - "rollup": "^2.45.2", - "rollup-plugin-postcss": "^4.0.0", - "rollup-plugin-svelte": "^7.1.0", - "rollup-plugin-terser": "^7.0.2" + "vite-plugin-commonjs": "^0.10.4" }, "keywords": [ "svelte" From f8855bb0c103eb9be492a6e3a42d4bd2b1abc084 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Nov 2024 10:59:34 +0100 Subject: [PATCH 091/192] Replace rollup.config for vite.config --- packages/bbui/rollup.config.js | 32 -------------------------------- packages/bbui/vite.config.js | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 32 deletions(-) delete mode 100644 packages/bbui/rollup.config.js create mode 100644 packages/bbui/vite.config.js diff --git a/packages/bbui/rollup.config.js b/packages/bbui/rollup.config.js deleted file mode 100644 index da274e0ba5..0000000000 --- a/packages/bbui/rollup.config.js +++ /dev/null @@ -1,32 +0,0 @@ -import svelte from "rollup-plugin-svelte" -import resolve from "@rollup/plugin-node-resolve" -import commonjs from "@rollup/plugin-commonjs" -import json from "@rollup/plugin-json" -import { terser } from "rollup-plugin-terser" -import postcss from "rollup-plugin-postcss" - -export default { - input: "src/index.js", - output: { - sourcemap: true, - format: "esm", - file: "dist/bbui.es.js", - }, - onwarn(warning, warn) { - // suppress eval warnings - if (warning.code === "EVAL") { - return - } - warn(warning) - }, - plugins: [ - resolve(), - commonjs(), - svelte({ - emitCss: true, - }), - postcss(), - terser(), - json(), - ], -} diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js new file mode 100644 index 0000000000..3d8afe8c07 --- /dev/null +++ b/packages/bbui/vite.config.js @@ -0,0 +1,29 @@ +import { defineConfig } from "vite" +import { svelte } from "@sveltejs/vite-plugin-svelte" +import postcss from "postcss" + +export default defineConfig({ + // https://vite.dev/guide/dep-pre-bundling#monorepos-and-linked-dependencies + optimizeDeps: { + include: ["shared-core"], + }, + build: { + sourcemap: true, + lib: { + entry: "src/index.js", + formats: ["es"], + fileName: "bbui.es", + }, + commonjsOptions: { + include: [/shared-core/, /node_modules/], + }, + }, + plugins: [ + svelte({ + emitCss: true, + }), + ], + css: { + postcss, + }, +}) From dee85857f9d1ba1e88324400c9e8b5d5b758eee8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Nov 2024 11:12:39 +0100 Subject: [PATCH 092/192] bbui use alias instead of packages --- packages/bbui/vite.config.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js index 3d8afe8c07..4c7da1c43e 100644 --- a/packages/bbui/vite.config.js +++ b/packages/bbui/vite.config.js @@ -1,12 +1,9 @@ import { defineConfig } from "vite" import { svelte } from "@sveltejs/vite-plugin-svelte" import postcss from "postcss" +import path from "path" export default defineConfig({ - // https://vite.dev/guide/dep-pre-bundling#monorepos-and-linked-dependencies - optimizeDeps: { - include: ["shared-core"], - }, build: { sourcemap: true, lib: { @@ -14,9 +11,6 @@ export default defineConfig({ formats: ["es"], fileName: "bbui.es", }, - commonjsOptions: { - include: [/shared-core/, /node_modules/], - }, }, plugins: [ svelte({ @@ -26,4 +20,10 @@ export default defineConfig({ css: { postcss, }, + resolve: { + alias: { + "@budibase/shared-core": path.resolve(__dirname, "../shared-core/src"), + "@budibase/types": path.resolve(__dirname, "../types/src"), + }, + }, }) From 18cd0169d848f0a46be732efe9b31f6737ec4d8f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 27 Nov 2024 11:37:51 +0100 Subject: [PATCH 093/192] Remove unused nx dependency --- packages/bbui/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 02cb81c2c4..28f2dfce13 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -91,8 +91,7 @@ "dependsOn": [ { "projects": [ - "@budibase/string-templates", - "@budibase/shared-core" + "@budibase/string-templates" ], "target": "build" } From a73165470e5ad3c93480829662dbe8d8a318d6ac Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 10:31:44 +0100 Subject: [PATCH 094/192] Only build sourcemap for !prod --- packages/bbui/vite.config.js | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js index 4c7da1c43e..b8a997dc4d 100644 --- a/packages/bbui/vite.config.js +++ b/packages/bbui/vite.config.js @@ -3,27 +3,30 @@ import { svelte } from "@sveltejs/vite-plugin-svelte" import postcss from "postcss" import path from "path" -export default defineConfig({ - build: { - sourcemap: true, - lib: { - entry: "src/index.js", - formats: ["es"], - fileName: "bbui.es", +export default defineConfig(({ mode }) => { + const isProduction = mode === "production" + return { + build: { + sourcemap: !isProduction, + lib: { + entry: "src/index.js", + formats: ["es"], + fileName: "bbui.es", + }, }, - }, - plugins: [ - svelte({ - emitCss: true, - }), - ], - css: { - postcss, - }, - resolve: { - alias: { - "@budibase/shared-core": path.resolve(__dirname, "../shared-core/src"), - "@budibase/types": path.resolve(__dirname, "../types/src"), + plugins: [ + svelte({ + emitCss: true, + }), + ], + css: { + postcss, }, - }, + resolve: { + alias: { + "@budibase/shared-core": path.resolve(__dirname, "../shared-core/src"), + "@budibase/types": path.resolve(__dirname, "../types/src"), + }, + }, + } }) From a7c2dd8171f880950dc3df6866cdfc13341a316f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 28 Nov 2024 10:36:13 +0100 Subject: [PATCH 095/192] Clean unused post-css --- packages/bbui/package.json | 4 +--- packages/bbui/vite.config.js | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 28f2dfce13..1720cc7ca8 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -17,9 +17,7 @@ "build": "vite build" }, "devDependencies": { - "@sveltejs/vite-plugin-svelte": "1.4.0", - "postcss": "^8.2.9", - "vite-plugin-commonjs": "^0.10.4" + "@sveltejs/vite-plugin-svelte": "1.4.0" }, "keywords": [ "svelte" diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js index b8a997dc4d..a084b16262 100644 --- a/packages/bbui/vite.config.js +++ b/packages/bbui/vite.config.js @@ -1,6 +1,5 @@ import { defineConfig } from "vite" import { svelte } from "@sveltejs/vite-plugin-svelte" -import postcss from "postcss" import path from "path" export default defineConfig(({ mode }) => { @@ -19,9 +18,6 @@ export default defineConfig(({ mode }) => { emitCss: true, }), ], - css: { - postcss, - }, resolve: { alias: { "@budibase/shared-core": path.resolve(__dirname, "../shared-core/src"), From 687afc61ad823c13f627cd07c0bac9994a7f6938 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 09:35:02 +0100 Subject: [PATCH 096/192] Update yarn.lock --- yarn.lock | 586 ++---------------------------------------------------- 1 file changed, 12 insertions(+), 574 deletions(-) diff --git a/yarn.lock b/yarn.lock index f52cdc79a5..6bc709e45d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4024,19 +4024,6 @@ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== -"@rollup/plugin-commonjs@^16.0.0": - version "16.0.0" - resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f" - integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw== - dependencies: - "@rollup/pluginutils" "^3.1.0" - commondir "^1.0.1" - estree-walker "^2.0.1" - glob "^7.1.6" - is-reference "^1.2.1" - magic-string "^0.25.7" - resolve "^1.17.0" - "@rollup/plugin-commonjs@^17.1.0": version "17.1.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-17.1.0.tgz#757ec88737dffa8aa913eb392fade2e45aef2a2d" @@ -4078,18 +4065,6 @@ dependencies: "@rollup/pluginutils" "^3.0.8" -"@rollup/plugin-node-resolve@^11.2.1": - version "11.2.1" - resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-11.2.1.tgz#82aa59397a29cd4e13248b106e6a4a1880362a60" - integrity sha512-yc2n43jcqVyGE2sqV5/YCmocy9ArjVAP/BeXyTtADTBBX6V0e5UMqwO8CdQ0kzjb6zu5P1qMzsScCMRvE9OlVg== - dependencies: - "@rollup/pluginutils" "^3.1.0" - "@types/resolve" "1.17.1" - builtin-modules "^3.1.0" - deepmerge "^4.2.2" - is-module "^1.0.0" - resolve "^1.19.0" - "@rollup/plugin-node-resolve@^15.2.3": version "15.2.3" resolved "https://registry.yarnpkg.com/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.2.3.tgz#e5e0b059bd85ca57489492f295ce88c2d4b0daf9" @@ -4136,14 +4111,6 @@ estree-walker "^1.0.1" picomatch "^2.2.2" -"@rollup/pluginutils@^4.1.0": - version "4.2.1" - resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-4.2.1.tgz#e6c6c3aba0744edce3fb2074922d3776c0af2a6d" - integrity sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ== - dependencies: - estree-walker "^2.0.1" - picomatch "^2.2.2" - "@rollup/pluginutils@^5.0.1": version "5.0.2" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-5.0.2.tgz#012b8f53c71e4f6f9cb317e311df1404f56e7a33" @@ -5301,11 +5268,6 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== -"@trysound/sax@0.2.0": - version "0.2.0" - resolved "https://registry.yarnpkg.com/@trysound/sax/-/sax-0.2.0.tgz#cccaab758af56761eb7bf37af6f03f326dd798ad" - integrity sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA== - "@tsconfig/node10@^1.0.7": version "1.0.9" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" @@ -6064,13 +6026,6 @@ dependencies: "@types/node" "*" -"@types/resolve@1.17.1": - version "1.17.1" - resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.17.1.tgz#3afd6ad8967c77e4376c598a82ddd58f46ec45d6" - integrity sha512-yy7HuzQhj0dhGpD8RLXSZWEkLsV9ibvxvi6EiJ3bkqLAO1RGo0WbkWQiwpRlSFymTJRz0d3k5LM3kkx8ArDbLw== - dependencies: - "@types/node" "*" - "@types/resolve@1.20.2": version "1.20.2" resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.2.tgz#97d26e00cd4a0423b4af620abecf3e6f442b7975" @@ -7608,11 +7563,6 @@ bn.js@^5.0.0, bn.js@^5.2.1: resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.1.tgz#0bc527a6a0d18d0aa8d5b0538ce4a77dccfa7b70" integrity sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ== -boolbase@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" - integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== - boolean@^3.0.1: version "3.2.0" resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.2.0.tgz#9e5294af4e98314494cbb17979fa54ca159f116b" @@ -7738,7 +7688,7 @@ browserify-sign@^4.0.0: readable-stream "^2.3.8" safe-buffer "^5.2.1" -browserslist@^4.0.0, browserslist@^4.21.3, browserslist@^4.21.4, browserslist@^4.21.5: +browserslist@^4.21.3, browserslist@^4.21.5: version "4.21.5" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.5.tgz#75c5dae60063ee641f977e00edd3cfb2fb7af6a7" integrity sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w== @@ -8041,17 +7991,7 @@ camelcase@^6.2.0, camelcase@^6.3.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-api@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0" - integrity sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw== - dependencies: - browserslist "^4.0.0" - caniuse-lite "^1.0.0" - lodash.memoize "^4.1.2" - lodash.uniq "^4.5.0" - -caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001449: +caniuse-lite@^1.0.30001449: version "1.0.30001460" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001460.tgz#31d2e26f0a2309860ed3eff154e03890d9d851a7" integrity sha512-Bud7abqjvEjipUkpLs4D7gR0l8hBYBHoa+tGtKJHvT2AYzLp1z7EmVkUT4ERpVUfca8S2HGIVs883D8pUH1ZzQ== @@ -8414,11 +8354,6 @@ color@^3.1.3: color-convert "^1.9.3" color-string "^1.6.0" -colord@^2.9.1: - version "2.9.3" - resolved "https://registry.yarnpkg.com/colord/-/colord-2.9.3.tgz#4f8ce919de456f1d5c1c368c307fe20f3e59fb43" - integrity sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw== - colorette@2.0.19, colorette@^2.0.7: version "2.0.19" resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798" @@ -8573,13 +8508,6 @@ concat-stream@^2.0.0: readable-stream "^3.0.2" typedarray "^0.0.6" -concat-with-sourcemaps@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/concat-with-sourcemaps/-/concat-with-sourcemaps-1.1.0.tgz#d4ea93f05ae25790951b99e7b3b09e3908a4082e" - integrity sha512-4gEjHJFT9e+2W/77h/DS5SGUgwDaOwprX8L/gl5+3ixnzkVJJsZWDSelmN3Oilw3LNDZjZV0yqH1hLG3k6nghg== - dependencies: - source-map "^0.6.1" - condense-newlines@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/condense-newlines/-/condense-newlines-0.2.1.tgz#3de985553139475d32502c83b02f60684d24c55f" @@ -8982,30 +8910,6 @@ crypto-randomuuid@^1.0.0: resolved "https://registry.yarnpkg.com/crypto-randomuuid/-/crypto-randomuuid-1.0.0.tgz#acf583e5e085e867ae23e107ff70279024f9e9e7" integrity sha512-/RC5F4l1SCqD/jazwUF6+t34Cd8zTSAGZ7rvvZu1whZUhD2a5MOGKjSGowoGcpj/cbVZk1ZODIooJEQQq3nNAA== -css-declaration-sorter@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/css-declaration-sorter/-/css-declaration-sorter-6.3.1.tgz#be5e1d71b7a992433fb1c542c7a1b835e45682ec" - integrity sha512-fBffmak0bPAnyqc/HO8C3n2sHrp9wcqQz6ES9koRF2/mLOVAx9zIQ3Y7R29sYCteTPqMCwns4WYQoCX91Xl3+w== - -css-select@^4.1.3: - version "4.3.0" - resolved "https://registry.yarnpkg.com/css-select/-/css-select-4.3.0.tgz#db7129b2846662fd8628cfc496abb2b59e41529b" - integrity sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ== - dependencies: - boolbase "^1.0.0" - css-what "^6.0.1" - domhandler "^4.3.1" - domutils "^2.8.0" - nth-check "^2.0.1" - -css-tree@^1.1.2, css-tree@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-1.1.3.tgz#eb4870fb6fd7707327ec95c2ff2ab09b5e8db91d" - integrity sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q== - dependencies: - mdn-data "2.0.14" - source-map "^0.6.1" - css-tree@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/css-tree/-/css-tree-2.3.1.tgz#10264ce1e5442e8572fc82fbe490644ff54b5c20" @@ -9014,11 +8918,6 @@ css-tree@^2.3.1: mdn-data "2.0.30" source-map-js "^1.0.1" -css-what@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.1.0.tgz#fb5effcf76f1ddea2c81bdfaa4de44e79bac70f4" - integrity sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw== - css.escape@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/css.escape/-/css.escape-1.5.1.tgz#42e27d4fa04ae32f931a4b4d4191fa9cddee97cb" @@ -9029,62 +8928,6 @@ cssesc@^3.0.0: resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee" integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg== -cssnano-preset-default@^5.2.14: - version "5.2.14" - resolved "https://registry.yarnpkg.com/cssnano-preset-default/-/cssnano-preset-default-5.2.14.tgz#309def4f7b7e16d71ab2438052093330d9ab45d8" - integrity sha512-t0SFesj/ZV2OTylqQVOrFgEh5uanxbO6ZAdeCrNsUQ6fVuXwYTxJPNAGvGTxHbD68ldIJNec7PyYZDBrfDQ+6A== - dependencies: - css-declaration-sorter "^6.3.1" - cssnano-utils "^3.1.0" - postcss-calc "^8.2.3" - postcss-colormin "^5.3.1" - postcss-convert-values "^5.1.3" - postcss-discard-comments "^5.1.2" - postcss-discard-duplicates "^5.1.0" - postcss-discard-empty "^5.1.1" - postcss-discard-overridden "^5.1.0" - postcss-merge-longhand "^5.1.7" - postcss-merge-rules "^5.1.4" - postcss-minify-font-values "^5.1.0" - postcss-minify-gradients "^5.1.1" - postcss-minify-params "^5.1.4" - postcss-minify-selectors "^5.2.1" - postcss-normalize-charset "^5.1.0" - postcss-normalize-display-values "^5.1.0" - postcss-normalize-positions "^5.1.1" - postcss-normalize-repeat-style "^5.1.1" - postcss-normalize-string "^5.1.0" - postcss-normalize-timing-functions "^5.1.0" - postcss-normalize-unicode "^5.1.1" - postcss-normalize-url "^5.1.0" - postcss-normalize-whitespace "^5.1.1" - postcss-ordered-values "^5.1.3" - postcss-reduce-initial "^5.1.2" - postcss-reduce-transforms "^5.1.0" - postcss-svgo "^5.1.0" - postcss-unique-selectors "^5.1.1" - -cssnano-utils@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/cssnano-utils/-/cssnano-utils-3.1.0.tgz#95684d08c91511edfc70d2636338ca37ef3a6861" - integrity sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA== - -cssnano@^5.0.1: - version "5.1.15" - resolved "https://registry.yarnpkg.com/cssnano/-/cssnano-5.1.15.tgz#ded66b5480d5127fcb44dac12ea5a983755136bf" - integrity sha512-j+BKgDcLDQA+eDifLx0EO4XSA56b7uut3BQFH+wbSaSTuGLuiyTa/wbRYthUXX8LC9mLg+WWKe8h+qJuwTAbHw== - dependencies: - cssnano-preset-default "^5.2.14" - lilconfig "^2.0.3" - yaml "^1.10.2" - -csso@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/csso/-/csso-4.2.0.tgz#ea3a561346e8dc9f546d6febedd50187cf389529" - integrity sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA== - dependencies: - css-tree "^1.1.2" - cssom@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" @@ -9924,15 +9767,6 @@ dom-accessibility-api@^0.6.3: resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.6.3.tgz#993e925cc1d73f2c662e7d75dd5a5445259a8fd8" integrity sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w== -dom-serializer@^1.0.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-1.4.1.tgz#de5d41b1aea290215dc45a6dae8adcf1d32e2d30" - integrity sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag== - dependencies: - domelementtype "^2.0.1" - domhandler "^4.2.0" - entities "^2.0.0" - dom-serializer@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" @@ -9947,7 +9781,7 @@ dom-walk@^0.1.0: resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.2.tgz#0c548bef048f4d1f2a97249002236060daa3fd84" integrity sha512-6QvTW9mrGeIegrFXdtQi9pk7O/nSK6lSdXW2eqUspN5LWD7UTji2Fqw5V2YLjBpHEoU9Xl/eUWNpDeZvoyOv2w== -domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0: +domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== @@ -9966,13 +9800,6 @@ domexception@^4.0.0: dependencies: webidl-conversions "^7.0.0" -domhandler@^4.2.0, domhandler@^4.3.1: - version "4.3.1" - resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-4.3.1.tgz#8d792033416f59d68bc03a5aa7b018c1ca89279c" - integrity sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ== - dependencies: - domelementtype "^2.2.0" - domhandler@^5.0.1, domhandler@^5.0.2: version "5.0.3" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" @@ -9980,15 +9807,6 @@ domhandler@^5.0.1, domhandler@^5.0.2: dependencies: domelementtype "^2.3.0" -domutils@^2.8.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.8.0.tgz#4437def5db6e2d1f5d6ee859bd95ca7d02048135" - integrity sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A== - dependencies: - dom-serializer "^1.0.1" - domelementtype "^2.2.0" - domhandler "^4.2.0" - domutils@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.0.1.tgz#696b3875238338cb186b6c0612bd4901c89a4f1c" @@ -10276,11 +10094,6 @@ enquirer@~2.3.6: dependencies: ansi-colors "^4.1.1" -entities@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" - integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== - entities@^4.2.0, entities@^4.3.0, entities@^4.4.0, entities@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" @@ -11577,13 +11390,6 @@ generate-function@^2.3.1: dependencies: is-property "^1.0.2" -generic-names@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/generic-names/-/generic-names-4.0.0.tgz#0bd8a2fd23fe8ea16cbd0a279acd69c06933d9a3" - integrity sha512-ySFolZQfw9FoDb3ed9d80Cm9f0+r7qj+HJkWjeD9RBfpxEVTlVhol+gvaQB/78WbwYfbnNh8nWHHBSlg072y6A== - dependencies: - loader-utils "^3.2.0" - generic-pool@3.9.0, generic-pool@^3.8.2: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" @@ -12548,16 +12354,6 @@ iconv-lite@0.6.3, iconv-lite@^0.6.2, iconv-lite@^0.6.3: dependencies: safer-buffer ">= 2.1.2 < 3.0.0" -icss-replace-symbols@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz#06ea6f83679a7749e386cfe1fe812ae5db223ded" - integrity sha512-chIaY3Vh2mh2Q3RGXttaDIzeiPvaVXJ+C4DAh/w3c37SKZ/U6PGMmuicR2EQQp9bKG8zLMCl7I+PtIoOOPp8Gg== - -icss-utils@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae" - integrity sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA== - idb-wrapper@^1.5.0: version "1.7.2" resolved "https://registry.yarnpkg.com/idb-wrapper/-/idb-wrapper-1.7.2.tgz#8251afd5e77fe95568b1c16152eb44b396767ea2" @@ -12621,13 +12417,6 @@ immediate@~3.0.5: resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== -import-cwd@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/import-cwd/-/import-cwd-3.0.0.tgz#20845547718015126ea9b3676b7592fb8bd4cf92" - integrity sha512-4pnzH16plW+hgvRECbDWpQl3cqtvSofHWh44met7ESfZ8UZOWWddm8hEyDTqREJ9RbYHY8gi8DqmaelApoOGMg== - dependencies: - import-from "^3.0.0" - import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" @@ -12636,13 +12425,6 @@ import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" -import-from@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/import-from/-/import-from-3.0.0.tgz#055cfec38cd5a27d8057ca51376d7d3bf0891966" - integrity sha512-CiuXOFFSzkU5x/CR0+z7T91Iht4CXgfCxVOFRhh2Zyhg5wOpWvvDLQUsWl+gcN+QscYBjez8hDCt85O7RLDttQ== - dependencies: - resolve-from "^5.0.0" - import-in-the-middle@1.11.2: version "1.11.2" resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz#dd848e72b63ca6cd7c34df8b8d97fc9baee6174f" @@ -14838,7 +14620,7 @@ lie@3.1.1: dependencies: immediate "~3.0.5" -lilconfig@^2.0.3, lilconfig@^2.0.5: +lilconfig@^2.0.5: version "2.1.0" resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52" integrity sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ== @@ -14878,11 +14660,6 @@ load-json-file@^4.0.0: pify "^3.0.0" strip-bom "^3.0.0" -loader-utils@^3.2.0: - version "3.2.1" - resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-3.2.1.tgz#4fb104b599daafd82ef3e1a41fb9265f87e1f576" - integrity sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw== - local-pkg@^0.4.2: version "0.4.3" resolved "https://registry.yarnpkg.com/local-pkg/-/local-pkg-0.4.3.tgz#0ff361ab3ae7f1c19113d9bb97b98b905dbc4963" @@ -15013,7 +14790,7 @@ lodash.keys@^4.2.0: resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-4.2.0.tgz#a08602ac12e4fb83f91fc1fb7a360a4d9ba35205" integrity sha512-J79MkJcp7Df5mizHiVNpjoHXLi4HLjh9VLS/M7lQSGoQ+0oQ+lWEigREkqKyizPB1IawvQLLKY8mzEcm1tkyxQ== -lodash.memoize@4.x, lodash.memoize@^4.1.2: +lodash.memoize@4.x: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" integrity sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag== @@ -15048,11 +14825,6 @@ lodash.union@^4.6.0: resolved "https://registry.yarnpkg.com/lodash.union/-/lodash.union-4.6.0.tgz#48bb5088409f16f1821666641c44dd1aaae3cd88" integrity sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw== -lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" - integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== - lodash.without@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac" @@ -15370,11 +15142,6 @@ md5.js@^1.3.4: inherits "^2.0.1" safe-buffer "^5.1.2" -mdn-data@2.0.14: - version "2.0.14" - resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" - integrity sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow== - mdn-data@2.0.30: version "2.0.30" resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.30.tgz#ce4df6f80af6cfbe218ecd5c552ba13c4dfa08cc" @@ -16207,11 +15974,6 @@ normalize-url@^4.1.0: resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-4.5.1.tgz#0dd90cf1288ee1d1313b87081c9a5932ee48518a" integrity sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA== -normalize-url@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" - integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== - notepack.io@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/notepack.io/-/notepack.io-3.0.1.tgz#2c2c9de1bd4e64a79d34e33c413081302a0d4019" @@ -16341,13 +16103,6 @@ npmlog@^6.0.0, npmlog@^6.0.2: gauge "^4.0.3" set-blocking "^2.0.0" -nth-check@^2.0.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" - integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== - dependencies: - boolbase "^1.0.0" - nunjucks@^3.2.3: version "3.2.4" resolved "https://registry.yarnpkg.com/nunjucks/-/nunjucks-3.2.4.tgz#f0878eef528ce7b0aa35d67cc6898635fd74649e" @@ -16861,7 +16616,7 @@ p-pipe@3.1.0: resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-3.1.0.tgz#48b57c922aa2e1af6a6404cb7c6bf0eb9cc8e60e" integrity sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw== -p-queue@6.6.2, p-queue@^6.6.2: +p-queue@6.6.2: version "6.6.2" resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== @@ -17317,7 +17072,7 @@ picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3, picomatc resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== -pify@5.0.0, pify@^5.0.0: +pify@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/pify/-/pify-5.0.0.tgz#1f5eca3f5e87ebec28cc6d54a0e4aaf00acc127f" integrity sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA== @@ -17509,53 +17264,7 @@ possible-typed-array-names@^1.0.0: resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz#89bb63c6fada2c3e90adc4a647beeeb39cc7bf8f" integrity sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q== -postcss-calc@^8.2.3: - version "8.2.4" - resolved "https://registry.yarnpkg.com/postcss-calc/-/postcss-calc-8.2.4.tgz#77b9c29bfcbe8a07ff6693dc87050828889739a5" - integrity sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q== - dependencies: - postcss-selector-parser "^6.0.9" - postcss-value-parser "^4.2.0" - -postcss-colormin@^5.3.1: - version "5.3.1" - resolved "https://registry.yarnpkg.com/postcss-colormin/-/postcss-colormin-5.3.1.tgz#86c27c26ed6ba00d96c79e08f3ffb418d1d1988f" - integrity sha512-UsWQG0AqTFQmpBegeLLc1+c3jIqBNB0zlDGRWR+dQ3pRKJL1oeMzyqmH3o2PIfn9MBdNrVPWhDbT769LxCTLJQ== - dependencies: - browserslist "^4.21.4" - caniuse-api "^3.0.0" - colord "^2.9.1" - postcss-value-parser "^4.2.0" - -postcss-convert-values@^5.1.3: - version "5.1.3" - resolved "https://registry.yarnpkg.com/postcss-convert-values/-/postcss-convert-values-5.1.3.tgz#04998bb9ba6b65aa31035d669a6af342c5f9d393" - integrity sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA== - dependencies: - browserslist "^4.21.4" - postcss-value-parser "^4.2.0" - -postcss-discard-comments@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/postcss-discard-comments/-/postcss-discard-comments-5.1.2.tgz#8df5e81d2925af2780075840c1526f0660e53696" - integrity sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ== - -postcss-discard-duplicates@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz#9eb4fe8456706a4eebd6d3b7b777d07bad03e848" - integrity sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw== - -postcss-discard-empty@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz#e57762343ff7f503fe53fca553d18d7f0c369c6c" - integrity sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A== - -postcss-discard-overridden@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz#7e8c5b53325747e9d90131bb88635282fb4a276e" - integrity sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw== - -postcss-load-config@^3.0.0, postcss-load-config@^3.1.4: +postcss-load-config@^3.1.4: version "3.1.4" resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-3.1.4.tgz#1ab2571faf84bb078877e1d07905eabe9ebda855" integrity sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg== @@ -17563,184 +17272,6 @@ postcss-load-config@^3.0.0, postcss-load-config@^3.1.4: lilconfig "^2.0.5" yaml "^1.10.2" -postcss-merge-longhand@^5.1.7: - version "5.1.7" - resolved "https://registry.yarnpkg.com/postcss-merge-longhand/-/postcss-merge-longhand-5.1.7.tgz#24a1bdf402d9ef0e70f568f39bdc0344d568fb16" - integrity sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ== - dependencies: - postcss-value-parser "^4.2.0" - stylehacks "^5.1.1" - -postcss-merge-rules@^5.1.4: - version "5.1.4" - resolved "https://registry.yarnpkg.com/postcss-merge-rules/-/postcss-merge-rules-5.1.4.tgz#2f26fa5cacb75b1402e213789f6766ae5e40313c" - integrity sha512-0R2IuYpgU93y9lhVbO/OylTtKMVcHb67zjWIfCiKR9rWL3GUk1677LAqD/BcHizukdZEjT8Ru3oHRoAYoJy44g== - dependencies: - browserslist "^4.21.4" - caniuse-api "^3.0.0" - cssnano-utils "^3.1.0" - postcss-selector-parser "^6.0.5" - -postcss-minify-font-values@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-minify-font-values/-/postcss-minify-font-values-5.1.0.tgz#f1df0014a726083d260d3bd85d7385fb89d1f01b" - integrity sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-minify-gradients@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-minify-gradients/-/postcss-minify-gradients-5.1.1.tgz#f1fe1b4f498134a5068240c2f25d46fcd236ba2c" - integrity sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw== - dependencies: - colord "^2.9.1" - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-minify-params@^5.1.4: - version "5.1.4" - resolved "https://registry.yarnpkg.com/postcss-minify-params/-/postcss-minify-params-5.1.4.tgz#c06a6c787128b3208b38c9364cfc40c8aa5d7352" - integrity sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw== - dependencies: - browserslist "^4.21.4" - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-minify-selectors@^5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/postcss-minify-selectors/-/postcss-minify-selectors-5.2.1.tgz#d4e7e6b46147b8117ea9325a915a801d5fe656c6" - integrity sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg== - dependencies: - postcss-selector-parser "^6.0.5" - -postcss-modules-extract-imports@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz#cda1f047c0ae80c97dbe28c3e76a43b88025741d" - integrity sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw== - -postcss-modules-local-by-default@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.0.tgz#ebbb54fae1598eecfdf691a02b3ff3b390a5a51c" - integrity sha512-sT7ihtmGSF9yhm6ggikHdV0hlziDTX7oFoXtuVWeDd3hHObNkcHRo9V3yg7vCAY7cONyxJC/XXCmmiHHcvX7bQ== - dependencies: - icss-utils "^5.0.0" - postcss-selector-parser "^6.0.2" - postcss-value-parser "^4.1.0" - -postcss-modules-scope@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz#9ef3151456d3bbfa120ca44898dfca6f2fa01f06" - integrity sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg== - dependencies: - postcss-selector-parser "^6.0.4" - -postcss-modules-values@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz#d7c5e7e68c3bb3c9b27cbf48ca0bb3ffb4602c9c" - integrity sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ== - dependencies: - icss-utils "^5.0.0" - -postcss-modules@^4.0.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/postcss-modules/-/postcss-modules-4.3.1.tgz#517c06c09eab07d133ae0effca2c510abba18048" - integrity sha512-ItUhSUxBBdNamkT3KzIZwYNNRFKmkJrofvC2nWab3CPKhYBQ1f27XXh1PAPE27Psx58jeelPsxWB/+og+KEH0Q== - dependencies: - generic-names "^4.0.0" - icss-replace-symbols "^1.1.0" - lodash.camelcase "^4.3.0" - postcss-modules-extract-imports "^3.0.0" - postcss-modules-local-by-default "^4.0.0" - postcss-modules-scope "^3.0.0" - postcss-modules-values "^4.0.0" - string-hash "^1.1.1" - -postcss-normalize-charset@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz#9302de0b29094b52c259e9b2cf8dc0879879f0ed" - integrity sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg== - -postcss-normalize-display-values@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-display-values/-/postcss-normalize-display-values-5.1.0.tgz#72abbae58081960e9edd7200fcf21ab8325c3da8" - integrity sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-positions@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-positions/-/postcss-normalize-positions-5.1.1.tgz#ef97279d894087b59325b45c47f1e863daefbb92" - integrity sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-repeat-style@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.1.1.tgz#e9eb96805204f4766df66fd09ed2e13545420fb2" - integrity sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-string@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-string/-/postcss-normalize-string-5.1.0.tgz#411961169e07308c82c1f8c55f3e8a337757e228" - integrity sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-timing-functions@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.1.0.tgz#d5614410f8f0b2388e9f240aa6011ba6f52dafbb" - integrity sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-normalize-unicode@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-unicode/-/postcss-normalize-unicode-5.1.1.tgz#f67297fca3fea7f17e0d2caa40769afc487aa030" - integrity sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA== - dependencies: - browserslist "^4.21.4" - postcss-value-parser "^4.2.0" - -postcss-normalize-url@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-normalize-url/-/postcss-normalize-url-5.1.0.tgz#ed9d88ca82e21abef99f743457d3729a042adcdc" - integrity sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew== - dependencies: - normalize-url "^6.0.1" - postcss-value-parser "^4.2.0" - -postcss-normalize-whitespace@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.1.1.tgz#08a1a0d1ffa17a7cc6efe1e6c9da969cc4493cfa" - integrity sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA== - dependencies: - postcss-value-parser "^4.2.0" - -postcss-ordered-values@^5.1.3: - version "5.1.3" - resolved "https://registry.yarnpkg.com/postcss-ordered-values/-/postcss-ordered-values-5.1.3.tgz#b6fd2bd10f937b23d86bc829c69e7732ce76ea38" - integrity sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ== - dependencies: - cssnano-utils "^3.1.0" - postcss-value-parser "^4.2.0" - -postcss-reduce-initial@^5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/postcss-reduce-initial/-/postcss-reduce-initial-5.1.2.tgz#798cd77b3e033eae7105c18c9d371d989e1382d6" - integrity sha512-dE/y2XRaqAi6OvjzD22pjTUQ8eOfc6m/natGHgKFBK9DxFmIm69YmaRVQrGgFlEfc1HePIurY0TmDeROK05rIg== - dependencies: - browserslist "^4.21.4" - caniuse-api "^3.0.0" - -postcss-reduce-transforms@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-reduce-transforms/-/postcss-reduce-transforms-5.1.0.tgz#333b70e7758b802f3dd0ddfe98bb1ccfef96b6e9" - integrity sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ== - dependencies: - postcss-value-parser "^4.2.0" - postcss-safe-parser@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/postcss-safe-parser/-/postcss-safe-parser-6.0.0.tgz#bb4c29894171a94bc5c996b9a30317ef402adaa1" @@ -17751,7 +17282,7 @@ postcss-scss@^4.0.8: resolved "https://registry.yarnpkg.com/postcss-scss/-/postcss-scss-4.0.9.tgz#a03c773cd4c9623cb04ce142a52afcec74806685" integrity sha512-AjKOeiwAitL/MXxQW2DliT28EKukvvbEWx3LBmJIRN8KfBGZbRTxNYW0kSqi1COiTZ57nZ9NW06S6ux//N1c9A== -postcss-selector-parser@^6.0.11, postcss-selector-parser@^6.0.2, postcss-selector-parser@^6.0.4, postcss-selector-parser@^6.0.5, postcss-selector-parser@^6.0.9: +postcss-selector-parser@^6.0.11: version "6.0.13" resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz#d05d8d76b1e8e173257ef9d60b706a8e5e99bf1b" integrity sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ== @@ -17759,26 +17290,6 @@ postcss-selector-parser@^6.0.11, postcss-selector-parser@^6.0.2, postcss-selecto cssesc "^3.0.0" util-deprecate "^1.0.2" -postcss-svgo@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-svgo/-/postcss-svgo-5.1.0.tgz#0a317400ced789f233a28826e77523f15857d80d" - integrity sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA== - dependencies: - postcss-value-parser "^4.2.0" - svgo "^2.7.0" - -postcss-unique-selectors@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz#a9f273d1eacd09e9aa6088f4b0507b18b1b541b6" - integrity sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA== - dependencies: - postcss-selector-parser "^6.0.5" - -postcss-value-parser@^4.1.0, postcss-value-parser@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514" - integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== - postcss-values-parser@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/postcss-values-parser/-/postcss-values-parser-2.0.1.tgz#da8b472d901da1e205b47bdc98637b9e9e550e5f" @@ -17797,7 +17308,7 @@ postcss-values-parser@^6.0.2: is-url-superb "^4.0.0" quote-unquote "^1.0.0" -postcss@^8.1.7, postcss@^8.2.9, postcss@^8.3.11, postcss@^8.4.12, postcss@^8.4.27, postcss@^8.4.29, postcss@^8.4.5: +postcss@^8.1.7, postcss@^8.3.11, postcss@^8.4.12, postcss@^8.4.27, postcss@^8.4.29, postcss@^8.4.5: version "8.4.41" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.41.tgz#d6104d3ba272d882fe18fc07d15dc2da62fa2681" integrity sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ== @@ -18283,11 +17794,6 @@ promise-retry@^2.0.1: err-code "^2.0.2" retry "^0.12.0" -promise.series@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/promise.series/-/promise.series-0.2.0.tgz#2cc7ebe959fc3a6619c04ab4dbdc9e452d864bbd" - integrity sha512-VWQJyU2bcDTgZw8kpfBpB/ejZASlCrzwz5f2hjb/zlujOEB4oeiAhHygAWq8ubsX2GVkD4kCU5V2dwOTaCY5EQ== - prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069" @@ -19024,7 +18530,7 @@ resolve.exports@^2.0.0: resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.0.tgz#c1a0028c2d166ec2fbf7d0644584927e76e7400e" integrity sha512-6K/gDlqgQscOlg9fSRpWstA8sYe8rbELsSTNpx+3kTrsVCzvSl0zIvRErM7fdl9ERWDsKnrLnwB+Ne89918XOg== -resolve@^1.10.0, resolve@^1.11.1, resolve@^1.14.2, resolve@^1.17.0, resolve@^1.19.0, resolve@^1.20.0, resolve@^1.21.0, resolve@^1.22.1, resolve@^1.22.3, resolve@^1.22.4: +resolve@^1.10.0, resolve@^1.11.1, resolve@^1.14.2, resolve@^1.17.0, resolve@^1.20.0, resolve@^1.21.0, resolve@^1.22.1, resolve@^1.22.3, resolve@^1.22.4: version "1.22.8" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== @@ -19158,33 +18664,6 @@ rollup-plugin-polyfill-node@^0.13.0: dependencies: "@rollup/plugin-inject" "^5.0.4" -rollup-plugin-postcss@^4.0.0: - version "4.0.2" - resolved "https://registry.yarnpkg.com/rollup-plugin-postcss/-/rollup-plugin-postcss-4.0.2.tgz#15e9462f39475059b368ce0e49c800fa4b1f7050" - integrity sha512-05EaY6zvZdmvPUDi3uCcAQoESDcYnv8ogJJQRp6V5kZ6J6P7uAVJlrTZcaaA20wTH527YTnKfkAoPxWI/jPp4w== - dependencies: - chalk "^4.1.0" - concat-with-sourcemaps "^1.1.0" - cssnano "^5.0.1" - import-cwd "^3.0.0" - p-queue "^6.6.2" - pify "^5.0.0" - postcss-load-config "^3.0.0" - postcss-modules "^4.0.0" - promise.series "^0.2.0" - resolve "^1.19.0" - rollup-pluginutils "^2.8.2" - safe-identifier "^0.4.2" - style-inject "^0.3.0" - -rollup-plugin-svelte@^7.1.0: - version "7.1.6" - resolved "https://registry.yarnpkg.com/rollup-plugin-svelte/-/rollup-plugin-svelte-7.1.6.tgz#44a4ea6c6e8ed976824d9fd40c78d048515e5838" - integrity sha512-nVFRBpGWI2qUY1OcSiEEA/kjCY2+vAjO9BI8SzA7NRrh2GTunLd6w2EYmnMt/atgdg8GvcNjLsmZmbQs/u4SQA== - dependencies: - "@rollup/pluginutils" "^4.1.0" - resolve.exports "^2.0.0" - rollup-plugin-terser@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/rollup-plugin-terser/-/rollup-plugin-terser-7.0.2.tgz#e8fbba4869981b2dc35ae7e8a502d5c6c04d324d" @@ -19207,7 +18686,7 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.8.1, rollup-pluginutils@^2.8.2: dependencies: estree-walker "^0.6.1" -rollup@^2.36.2, rollup@^2.45.2: +rollup@^2.36.2: version "2.79.1" resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7" integrity sha512-uKxbd0IhMZOhjAiD5oAFp7BqvkA4Dv47qpOCtaNvng4HBwdbWtdOh8f5nZNuk2rp51PMGk3bzfWu5oayNEuYnw== @@ -19314,11 +18793,6 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-identifier@^0.4.2: - version "0.4.2" - resolved "https://registry.yarnpkg.com/safe-identifier/-/safe-identifier-0.4.2.tgz#cf6bfca31c2897c588092d1750d30ef501d59fcb" - integrity sha512-6pNbSMW6OhAi9j+N8V+U715yBQsaWJ7eyEUaOrawX+isg5ZxhUlV1NipNtgaKHmFGiABwt+ZF04Ii+3Xjkg+8w== - safe-regex-test@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/safe-regex-test/-/safe-regex-test-1.0.3.tgz#a5b4c0f06e0ab50ea2c395c14d8371232924c377" @@ -20003,11 +19477,6 @@ ssri@^9.0.0, ssri@^9.0.1: dependencies: minipass "^3.1.1" -stable@^0.1.8: - version "0.1.8" - resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf" - integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w== - stack-trace@0.0.x: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" @@ -20106,11 +19575,6 @@ strict-uri-encode@^2.0.0: resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" integrity sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ== -string-hash@^1.1.1: - version "1.1.3" - resolved "https://registry.yarnpkg.com/string-hash/-/string-hash-1.1.3.tgz#e8aafc0ac1855b4666929ed7dd1275df5d6c811b" - integrity sha512-kJUvRUFK49aub+a7T1nNE66EJbZBMnBgoC1UbCZ5n6bsZKBRga4KgBRTMn/pFkeCZSYtNeSyMxPDM0AXWELk2A== - string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -20350,24 +19814,11 @@ stubs@^3.0.0: resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" integrity "sha1-6NK6H6nJBXAwPAMLaQD31fiavls= sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" -style-inject@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/style-inject/-/style-inject-0.3.0.tgz#d21c477affec91811cc82355832a700d22bf8dd3" - integrity sha512-IezA2qp+vcdlhJaVm5SOdPPTUu0FCEqfNSli2vRuSIBbu5Nq5UvygTk/VzeCqfLz2Atj3dVII5QBKGZRZ0edzw== - style-mod@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/style-mod/-/style-mod-4.0.3.tgz#136c4abc905f82a866a18b39df4dc08ec762b1ad" integrity sha512-78Jv8kYJdjbvRwwijtCevYADfsI0lGzYJe4mMFdceO8l75DFFDoqBhR1jVDicDRRaX4//g1u9wKeo+ztc2h1Rw== -stylehacks@^5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/stylehacks/-/stylehacks-5.1.1.tgz#7934a34eb59d7152149fa69d6e9e56f2fc34bcc9" - integrity sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw== - dependencies: - browserslist "^4.21.4" - postcss-selector-parser "^6.0.4" - stylus-lookup@^3.0.1: version "3.0.2" resolved "https://registry.yarnpkg.com/stylus-lookup/-/stylus-lookup-3.0.2.tgz#c9eca3ff799691020f30b382260a67355fefdddd" @@ -20600,19 +20051,6 @@ svg.select.js@^3.0.1: dependencies: svg.js "^2.6.5" -svgo@^2.7.0: - version "2.8.0" - resolved "https://registry.yarnpkg.com/svgo/-/svgo-2.8.0.tgz#4ff80cce6710dc2795f0c7c74101e6764cfccd24" - integrity sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg== - dependencies: - "@trysound/sax" "0.2.0" - commander "^7.2.0" - css-select "^4.1.3" - css-tree "^1.1.3" - csso "^4.2.0" - picocolors "^1.0.0" - stable "^0.1.8" - swagger-jsdoc@6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/swagger-jsdoc/-/swagger-jsdoc-6.1.0.tgz#c2b86321f2c4dde8947b418fe8a4bc94431d5522" From 6221922b3322bf735b36f7a4f573d842418311b7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 09:38:18 +0100 Subject: [PATCH 097/192] Update bundle refs --- packages/bbui/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 1720cc7ca8..aa36474aed 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -4,10 +4,10 @@ "version": "0.0.0", "license": "MPL-2.0", "svelte": "src/index.js", - "module": "dist/bbui.es.js", + "module": "dist/bbui.es.mjs", "exports": { ".": { - "import": "./dist/bbui.es.js" + "import": "./dist/bbui.es.mjs" }, "./package.json": "./package.json", "./spectrum-icons-rollup.js": "./src/spectrum-icons-rollup.js", From 115b1db968ec1be5125637d08f22a6d59e8d2f58 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 09:40:16 +0100 Subject: [PATCH 098/192] Add css into the bundle --- packages/bbui/package.json | 3 ++- packages/bbui/vite.config.js | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index aa36474aed..c8732aca63 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -17,7 +17,8 @@ "build": "vite build" }, "devDependencies": { - "@sveltejs/vite-plugin-svelte": "1.4.0" + "@sveltejs/vite-plugin-svelte": "1.4.0", + "vite-plugin-css-injected-by-js": "3.5.2" }, "keywords": [ "svelte" diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js index a084b16262..997c616fd5 100644 --- a/packages/bbui/vite.config.js +++ b/packages/bbui/vite.config.js @@ -1,6 +1,7 @@ import { defineConfig } from "vite" import { svelte } from "@sveltejs/vite-plugin-svelte" import path from "path" +import cssInjectedByJsPlugin from "vite-plugin-css-injected-by-js" export default defineConfig(({ mode }) => { const isProduction = mode === "production" @@ -17,6 +18,7 @@ export default defineConfig(({ mode }) => { svelte({ emitCss: true, }), + cssInjectedByJsPlugin(), ], resolve: { alias: { From ef5ecb565a45e99cb78e28defb2487e75aea5e14 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 09:43:04 +0100 Subject: [PATCH 099/192] `check-lockfile` only for non-oss --- .github/workflows/budibase_ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 1258bddcca..90b747f07e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -281,6 +281,7 @@ jobs: check-lockfile: runs-on: ubuntu-latest + if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo uses: actions/checkout@v4 From 6f8f866b7e39761ad96f5ecf4036dc651c507979 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 28 Nov 2024 23:37:20 +0530 Subject: [PATCH 100/192] fix: loop context lost when max iterations are reached. Add/test cases for the same --- .../tests/scenarios/looping.spec.ts | 38 +++++++++++++++++++ packages/server/src/constants/index.ts | 1 - packages/server/src/threads/automation.ts | 20 +++------- .../documents/app/automation/automation.ts | 1 + 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9c5313e9da..66614e716a 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -152,6 +152,44 @@ describe("Loop automations", () => { ) }) + it("ensure the loop stops if the max iterations are reached", async () => { + const builder = createAutomationBuilder({ + name: "Test Loop max iterations", + }) + + const results = await builder + .appAction({ fields: {} }) + .loop({ + option: LoopStepType.ARRAY, + binding: ["test", "test2", "test3"], + iterations: 2, + }) + .serverLog({ text: "{{loop.currentItem}}" }) + .serverLog({ text: "{{steps.1.iterations}}" }) + .run() + + expect(results.steps[1].outputs.iterations).toBe(2) + }) + + it("should run an automation with loop and max iterations to ensure context correctness further down the tree", async () => { + const builder = createAutomationBuilder({ + name: "Test context down tree with Loop and max iterations", + }) + + const results = await builder + .appAction({ fields: {} }) + .loop({ + option: LoopStepType.ARRAY, + binding: ["test", "test2", "test3"], + iterations: 2, + }) + .serverLog({ text: "{{loop.currentItem}}" }) + .serverLog({ text: "{{steps.1.iterations}}" }) + .run() + + expect(results.steps[2].outputs.message).toContain("- 2") + }) + it("should run an automation where a loop is successfully run twice", async () => { const builder = createAutomationBuilder({ name: "Test Trigger with Loop and Create Row", diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 604a81cd9f..89e2f26516 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -137,7 +137,6 @@ export enum InvalidColumns { export enum AutomationErrors { INCORRECT_TYPE = "INCORRECT_TYPE", - MAX_ITERATIONS = "MAX_ITERATIONS_REACHED", FAILURE_CONDITION = "FAILURE_CONDITION_MET", } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 33e11a6ac0..051b485a54 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -392,6 +392,7 @@ class Orchestrator { let iterationCount = 0 let shouldCleanup = true + let reachedMaxIterations = false for (let loopStepIndex = 0; loopStepIndex < iterations; loopStepIndex++) { try { @@ -419,19 +420,8 @@ class Orchestrator { loopStepIndex === env.AUTOMATION_MAX_ITERATIONS || (loopStep.inputs.iterations && loopStepIndex === maxIterations) ) { - this.updateContextAndOutput( - pathStepIdx + 1, - steps[stepToLoopIndex], - { - items: this.loopStepOutputs, - iterations: loopStepIndex, - }, - { - status: AutomationErrors.MAX_ITERATIONS, - success: true, - } - ) - shouldCleanup = false + reachedMaxIterations = true + shouldCleanup = true break } @@ -480,7 +470,9 @@ class Orchestrator { success: true, } : { - success: true, + success: reachedMaxIterations + ? AutomationStepStatus.MAX_ITERATIONS + : true, items: this.loopStepOutputs, iterations: iterationCount, } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 1af892d8d1..71530c7939 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -174,6 +174,7 @@ export enum AutomationFeature { export enum AutomationStepStatus { NO_ITERATIONS = "no_iterations", + MAX_ITERATIONS = "max_iterations_reached", } export enum AutomationStatus { From 26f2c69fb94e8fdb065000983dc94f65253a610d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 29 Nov 2024 15:30:20 +0530 Subject: [PATCH 101/192] fix PR comment for keeping status boolean and adding status conditionally --- packages/server/src/threads/automation.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 051b485a54..7e8f947580 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -470,13 +470,15 @@ class Orchestrator { success: true, } : { - success: reachedMaxIterations - ? AutomationStepStatus.MAX_ITERATIONS - : true, + success: true, items: this.loopStepOutputs, iterations: iterationCount, } + if (reachedMaxIterations && iterations !== 0) { + tempOutput.status = AutomationStepStatus.MAX_ITERATIONS + } + // Loop Step clean up this.executionOutput.steps.splice(pathStepIdx, 0, { id: steps[stepToLoopIndex].id, From 5a524be591b99db1953a674d52fa84c0489e1ddd Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 11:03:13 +0100 Subject: [PATCH 102/192] dist/bbui.es.mjs -> dist/bbui.mjs --- packages/bbui/package.json | 4 ++-- packages/bbui/vite.config.js | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index c8732aca63..35f6676fdc 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -4,10 +4,10 @@ "version": "0.0.0", "license": "MPL-2.0", "svelte": "src/index.js", - "module": "dist/bbui.es.mjs", + "module": "dist/bbui.mjs", "exports": { ".": { - "import": "./dist/bbui.es.mjs" + "import": "./dist/bbui.mjs" }, "./package.json": "./package.json", "./spectrum-icons-rollup.js": "./src/spectrum-icons-rollup.js", diff --git a/packages/bbui/vite.config.js b/packages/bbui/vite.config.js index 997c616fd5..bf0f9fc26d 100644 --- a/packages/bbui/vite.config.js +++ b/packages/bbui/vite.config.js @@ -11,7 +11,6 @@ export default defineConfig(({ mode }) => { lib: { entry: "src/index.js", formats: ["es"], - fileName: "bbui.es", }, }, plugins: [ From f83ce2fca16838d4b7dc12e27e73256eb74e0b39 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 29 Nov 2024 10:30:06 +0000 Subject: [PATCH 103/192] Bump version to 3.2.15 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index be6673e7ae..cf93c66be9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.14", + "version": "3.2.15", "npmClient": "yarn", "concurrency": 20, "command": { From 16030cbb39c3bf51dee77c277d367f18e088feeb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 11:34:54 +0100 Subject: [PATCH 104/192] Add missing script --- scripts/syncLocalDependencies.js | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 scripts/syncLocalDependencies.js diff --git a/scripts/syncLocalDependencies.js b/scripts/syncLocalDependencies.js new file mode 100755 index 0000000000..1c402e9a44 --- /dev/null +++ b/scripts/syncLocalDependencies.js @@ -0,0 +1,60 @@ +const fs = require("fs") +const path = require("path") +const { execSync } = require("child_process") + +// Get the version argument from the command line +const version = process.argv[2] +if (!version) { + console.error("Usage: node update-workspace-dependencies.js ") + process.exit(1) +} + +// Get the list of workspaces with mismatched dependencies +const output = execSync("yarn --silent workspaces info --json", { + encoding: "utf-8", +}) +const data = JSON.parse(output) + +const workspaces = Object.keys(data).filter(key => { + return data[key].mismatchedWorkspaceDependencies?.length +}) + +// Loop through each workspace and update the dependencies +workspaces.forEach(workspace => { + const dependencies = data[workspace].mismatchedWorkspaceDependencies + + // Loop through each dependency and update its version in package.json + const packageJsonPath = path.join(data[workspace].location, "package.json") + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8")) + let hasChanges = false + + dependencies.forEach(dependency => { + if (packageJson.dependencies?.[dependency]) { + packageJson.dependencies[dependency] = version + hasChanges = true + } + if (packageJson.devDependencies?.[dependency]) { + packageJson.devDependencies[dependency] = version + hasChanges = true + } + if (packageJson.peerDependencies?.[dependency]) { + packageJson.peerDependencies[dependency] = version + hasChanges = true + } + }) + + // Write changes to package.json if there are any + if (hasChanges) { + fs.writeFileSync( + packageJsonPath, + JSON.stringify(packageJson, null, 2) + "\n" + ) + } +}) + +const rootPackageJson = JSON.parse(fs.readFileSync("package.json", "utf-8")) +delete rootPackageJson["resolutions"] +fs.writeFileSync( + "package.json", + JSON.stringify(rootPackageJson, null, 2) + "\n" +) From d2f7694afaeb5b24b716bf38dc6110c720dc4f88 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 29 Nov 2024 10:36:19 +0000 Subject: [PATCH 105/192] Bump version to 3.2.16 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index cf93c66be9..74e0cde617 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.15", + "version": "3.2.16", "npmClient": "yarn", "concurrency": 20, "command": { From 79d89e5f0ccc1d7c3837690048955e2e6d83fbc2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 12:17:46 +0100 Subject: [PATCH 106/192] Update pro for oss --- package.json | 4 ++-- yarn.lock | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 31e4110a35..2e9a2cc9af 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@types/node": "20.10.0", "@types/proper-lockfile": "^4.1.4", "@typescript-eslint/parser": "6.9.0", + "cross-spawn": "7.0.6", "depcheck": "^1.4.7", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.14.0", @@ -29,8 +30,7 @@ "svelte-eslint-parser": "^0.33.1", "typescript": "5.5.2", "typescript-eslint": "^7.3.1", - "yargs": "^17.7.2", - "cross-spawn": "7.0.6" + "yargs": "^17.7.2" }, "scripts": { "get-past-client-version": "node scripts/getPastClientVersion.js", diff --git a/yarn.lock b/yarn.lock index 6bc709e45d..44781eab4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2196,7 +2196,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@3.2.14": +"@budibase/backend-core@3.2.16": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2280,15 +2280,15 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "3.2.14" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.14.tgz#2ea1ef4372803304ac3a06ce77620ce54a09dd02" - integrity sha512-ski7NeXOdDIjC0vapo+hxr8/VnbDhmOLF5TdvoLBgviMk7ij2rNPI2dNs+6kQuVXAEVJPD6bJ4Ke0z2d5YcYrQ== + version "3.2.16" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.2.16.tgz#571751c5ab288ecbdd14791e4dab207b9523150f" + integrity sha512-HRKfpyYk+fJ87rOAvuBw1EcBv33xliSEf/ww7Wmvzx/xdxas6a7ZHoblw0BU8tnb5Q2rmYTf61RlqOIC4iC+0Q== dependencies: "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "3.2.14" - "@budibase/shared-core" "3.2.14" - "@budibase/string-templates" "3.2.14" - "@budibase/types" "3.2.14" + "@budibase/backend-core" "3.2.16" + "@budibase/shared-core" "3.2.16" + "@budibase/string-templates" "3.2.16" + "@budibase/types" "3.2.16" "@koa/router" "13.1.0" bull "4.10.1" dd-trace "5.26.0" @@ -2301,13 +2301,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@3.2.14": +"@budibase/shared-core@3.2.16": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@3.2.14": +"@budibase/string-templates@3.2.16": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2315,7 +2315,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@3.2.14": +"@budibase/types@3.2.16": version "0.0.0" dependencies: scim-patch "^0.8.1" From a6022c11a5e1ab2826b4950d5fa23b2ef083e06f Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 29 Nov 2024 16:52:55 +0530 Subject: [PATCH 107/192] fix loop not counting as step, test case failing --- .../server/src/automations/tests/scenarios/looping.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 66614e716a..0baa69b3bc 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -168,7 +168,7 @@ describe("Loop automations", () => { .serverLog({ text: "{{steps.1.iterations}}" }) .run() - expect(results.steps[1].outputs.iterations).toBe(2) + expect(results.steps[0].outputs.iterations).toBe(2) }) it("should run an automation with loop and max iterations to ensure context correctness further down the tree", async () => { @@ -187,7 +187,7 @@ describe("Loop automations", () => { .serverLog({ text: "{{steps.1.iterations}}" }) .run() - expect(results.steps[2].outputs.message).toContain("- 2") + expect(results.steps[1].outputs.message).toContain("- 2") }) it("should run an automation where a loop is successfully run twice", async () => { From b48f2d83bad0082915cc0ea6c4a94ebf24a2f89a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 12:31:26 +0100 Subject: [PATCH 108/192] Add flag --- packages/types/src/sdk/featureFlag.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 64a7362e9f..387b7e884a 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -4,8 +4,8 @@ export enum FeatureFlag { AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING", AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS", DEFAULT_VALUES = "DEFAULT_VALUES", - BUDIBASE_AI = "BUDIBASE_AI", + USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR", } export interface TenantFeatureFlags { From 0ea5117a295cd6afd25cfcc0619149389da2f0f9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:04:09 +0100 Subject: [PATCH 109/192] Use record to provide typings --- packages/backend-core/src/features/features.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index b3f016e88a..7f53f9b7d3 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -266,12 +266,13 @@ export class FlagSet, T extends { [key: string]: V }> { // new flag, add it here and use the `fetch` and `get` functions to access it. // All of the machinery in this file is to make sure that flags have their // default values set correctly and their types flow through the system. -export const flags = new FlagSet({ +const flagsConfig: Record> = { [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true), [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true), [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true), -}) +} +export const flags = new FlagSet(flagsConfig) type UnwrapPromise = T extends Promise ? U : T export type FeatureFlags = UnwrapPromise> From 40f50710e56e442aca5b581bc749d33f262bebe5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 12:54:34 +0100 Subject: [PATCH 110/192] Clean old flags --- packages/types/src/sdk/featureFlag.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 387b7e884a..98e744324c 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,6 +1,4 @@ export enum FeatureFlag { - PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE", - PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT", AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING", AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS", DEFAULT_VALUES = "DEFAULT_VALUES", From 96916a469d112a19fe9b9a39d7620d0523b7bdec Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:05:03 +0100 Subject: [PATCH 111/192] Configure flag --- packages/backend-core/src/features/features.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 7f53f9b7d3..650254fcb2 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -271,6 +271,7 @@ const flagsConfig: Record> = { [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true), [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true), + [FeatureFlag.USE_ZOD_VALIDATOR]: Flag.boolean(env.isDev()), } export const flags = new FlagSet(flagsConfig) From ccbf282c69ef4f4373de6a54513d60c94f3e7f30 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:07:24 +0100 Subject: [PATCH 112/192] Flag zod --- packages/server/src/middleware/zod-validator.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/server/src/middleware/zod-validator.ts b/packages/server/src/middleware/zod-validator.ts index 0ad346afc3..e8cc2c470a 100644 --- a/packages/server/src/middleware/zod-validator.ts +++ b/packages/server/src/middleware/zod-validator.ts @@ -1,11 +1,16 @@ -import { Ctx } from "@budibase/types" +import { features } from "@budibase/backend-core" +import { Ctx, FeatureFlag } from "@budibase/types" import { AnyZodObject } from "zod" import { fromZodError } from "zod-validation-error" function validate(schema: AnyZodObject, property: "body" | "params") { // Return a Koa middleware function - return (ctx: Ctx, next: any) => { + return async (ctx: Ctx, next: any) => { + if (!(await features.flags.isEnabled(FeatureFlag.USE_ZOD_VALIDATOR))) { + return next() + } + if (!schema) { return next() } From a2f1977376762889ab9bee6c57611bd4650d3e85 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:12:30 +0100 Subject: [PATCH 113/192] Add validation for search view --- packages/server/src/api/routes/row.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index fc09b8c81d..61cdee0c2f 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -90,6 +90,7 @@ router router.post( "/api/v2/views/:viewId/search", internalSearchValidator(), + validateBody(searchRowRequestValidator), authorizedResource(PermissionType.VIEW, PermissionLevel.READ, "viewId"), rowController.views.searchView ) From 26be5740857eb2475fa7d51760494dbe201720d8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:15:32 +0100 Subject: [PATCH 114/192] Fix nullish --- packages/types/src/api/web/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/api/web/app/rows/search.ts b/packages/types/src/api/web/app/rows/search.ts index b74e490677..7ba23bceca 100644 --- a/packages/types/src/api/web/app/rows/search.ts +++ b/packages/types/src/api/web/app/rows/search.ts @@ -68,7 +68,7 @@ const searchRowRequest = z.object({ paginate: z.boolean().optional(), bookmark: z.union([z.string(), z.number()]).nullish(), limit: z.number().optional(), - sort: z.string().optional(), + sort: z.string().nullish(), sortOrder: z.nativeEnum(SortOrder).optional(), sortType: z.nativeEnum(SortType).nullish(), version: z.string().optional(), From 57c466366360b034d5b59fe4fc346e7e18e44d87 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:49:43 +0100 Subject: [PATCH 115/192] Fix types --- packages/server/src/api/controllers/row/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 2dc9381183..cc5491e54f 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -258,7 +258,7 @@ export async function search(ctx: Ctx) { bookmark: searchRequest.bookmark ?? undefined, paginate: searchRequest.paginate, limit: searchRequest.limit, - sort: searchRequest.sort, + sort: searchRequest.sort ?? undefined, sortOrder: searchRequest.sortOrder, sortType: searchRequest.sortType ?? undefined, countRows: searchRequest.countRows, From a2730279d4e82a33496d4b60515e1d25356725d4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 29 Nov 2024 13:51:31 +0100 Subject: [PATCH 116/192] Fix types --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 8ba165ded0..c66197334e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -212,7 +212,7 @@ if (descriptions.length) { const inMemoryQuery: RequiredKeys< Omit > = { - sort: this.query.sort, + sort: this.query.sort ?? undefined, query: { ...this.query.query }, paginate: this.query.paginate, bookmark: this.query.bookmark ?? undefined, From 39859e10ed32c8c25f0f540087b858be18a3b135 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 29 Nov 2024 13:12:13 +0000 Subject: [PATCH 117/192] Don't fetch account for identifyTenantGroup posthog event (#15092) * Don't fetch account for identifyTenantGroup posthog event * Support self hosted * Fix types --- .../backend-core/src/events/identification.ts | 16 +--------------- .../src/migrations/functions/backfill/global.ts | 9 +++++---- .../worker/src/api/controllers/global/users.ts | 13 +++++-------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 69bf7009b2..6117f4b485 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -121,7 +121,7 @@ const identifyInstallationGroup = async ( const identifyTenantGroup = async ( tenantId: string, - account: Account | undefined, + hosting: Hosting, timestamp?: string | number ): Promise => { const id = await getEventTenantId(tenantId) @@ -129,26 +129,12 @@ const identifyTenantGroup = async ( const installationId = await getInstallationId() const environment = getDeploymentEnvironment() - let hosting: Hosting - let profession: string | undefined - let companySize: string | undefined - - if (account) { - profession = account.profession - companySize = account.size - hosting = account.hosting - } else { - hosting = getHostingFromEnv() - } - const group: TenantGroup = { id, type, hosting, environment, installationId, - profession, - companySize, } await identifyGroup(group, timestamp) diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index a56c92492b..7f718cee2f 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -9,11 +9,12 @@ import { db as dbUtils, } from "@budibase/backend-core" import { - QuotaUsage, - CloudAccount, App, - TenantBackfillSucceededEvent, + CloudAccount, Event, + Hosting, + QuotaUsage, + TenantBackfillSucceededEvent, User, } from "@budibase/types" import env from "../../../environment" @@ -125,7 +126,7 @@ export const run = async (db: any) => { try { await events.identification.identifyTenantGroup( tenantId, - account, + env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD, timestamp ) } catch (e) { diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 2479a50d9e..e977d5ff5d 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -6,12 +6,12 @@ import { AddSSoUserRequest, BulkUserRequest, BulkUserResponse, - CloudAccount, CreateAdminUserRequest, CreateAdminUserResponse, Ctx, DeleteInviteUserRequest, DeleteInviteUsersRequest, + Hosting, InviteUserRequest, InviteUsersRequest, InviteUsersResponse, @@ -26,7 +26,6 @@ import { UserIdentifier, } from "@budibase/types" import { - accounts, users, cache, ErrorCode, @@ -192,12 +191,10 @@ export const adminUser = async ( lastName: familyName, }) - // events - let account: CloudAccount | undefined - if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - account = await accounts.getAccountByTenantId(tenantId) - } - await events.identification.identifyTenantGroup(tenantId, account) + await events.identification.identifyTenantGroup( + tenantId, + env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD + ) ctx.body = { _id: finalUser._id!, From e4fd2c94d41ba2b8e5f1eec19700d8758af97e8d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 16:03:53 +0000 Subject: [PATCH 118/192] Api key, application and analytics API typing. --- .../server/src/api/controllers/analytics.ts | 14 ++++-- .../server/src/api/controllers/apikeys.ts | 23 +++++++--- .../server/src/api/controllers/application.ts | 44 ++++++++++++++----- .../src/api/controllers/deploy/index.ts | 11 ++++- .../server/src/sdk/app/applications/sync.ts | 2 +- packages/types/src/api/web/analytics.ts | 4 ++ packages/types/src/api/web/apikeys.ts | 10 +++++ packages/types/src/api/web/application.ts | 25 +++++++++++ packages/types/src/api/web/deployment.ts | 3 ++ packages/types/src/api/web/index.ts | 2 + .../types/src/documents/app/deployment.ts | 7 +++ packages/types/src/documents/app/index.ts | 1 + .../types/src/documents/global/apikeys.ts | 5 +++ packages/types/src/documents/global/index.ts | 1 + 14 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 packages/types/src/api/web/apikeys.ts create mode 100644 packages/types/src/api/web/deployment.ts create mode 100644 packages/types/src/documents/app/deployment.ts create mode 100644 packages/types/src/documents/global/apikeys.ts diff --git a/packages/server/src/api/controllers/analytics.ts b/packages/server/src/api/controllers/analytics.ts index f60c7b0961..8c87af7251 100644 --- a/packages/server/src/api/controllers/analytics.ts +++ b/packages/server/src/api/controllers/analytics.ts @@ -1,16 +1,22 @@ import { events, context } from "@budibase/backend-core" -import { AnalyticsPingRequest, App, PingSource } from "@budibase/types" +import { + AnalyticsPingRequest, + App, + PingSource, + Ctx, + AnalyticsEnabledResponse, +} from "@budibase/types" import { DocumentType, isDevAppID } from "../../db/utils" -export const isEnabled = async (ctx: any) => { +export const isEnabled = async (ctx: Ctx) => { const enabled = await events.analytics.enabled() ctx.body = { enabled, } } -export const ping = async (ctx: any) => { - const body = ctx.request.body as AnalyticsPingRequest +export const ping = async (ctx: Ctx) => { + const body = ctx.request.body switch (body.source) { case PingSource.APP: { diff --git a/packages/server/src/api/controllers/apikeys.ts b/packages/server/src/api/controllers/apikeys.ts index 2a02078483..8f841cd457 100644 --- a/packages/server/src/api/controllers/apikeys.ts +++ b/packages/server/src/api/controllers/apikeys.ts @@ -1,18 +1,25 @@ import { db as dbCore, tenancy } from "@budibase/backend-core" -import { BBContext, Document } from "@budibase/types" +import { + Document, + UserCtx, + ApiKeyDoc, + ApiKeyFetchResponse, + ApiKeyUpdateRequest, + ApiKeyUpdateResponse, +} from "@budibase/types" const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys async function getBuilderMainDoc() { const db = tenancy.getGlobalDB() - try { - return await db.get(KEYS_DOC) - } catch (err) { - // doesn't exist yet, nothing to get + const doc = await db.tryGet(KEYS_DOC) + if (!doc) { return { _id: KEYS_DOC, + apiKeys: {}, } } + return doc } async function setBuilderMainDoc(doc: Document) { @@ -22,7 +29,7 @@ async function setBuilderMainDoc(doc: Document) { return db.put(doc) } -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { try { const mainDoc = await getBuilderMainDoc() ctx.body = mainDoc.apiKeys ? mainDoc.apiKeys : {} @@ -32,7 +39,9 @@ export async function fetch(ctx: BBContext) { } } -export async function update(ctx: BBContext) { +export async function update( + ctx: UserCtx +) { const key = ctx.params.key const value = ctx.request.body.value diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 101257c321..9170ba54c6 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -59,6 +59,16 @@ import { BBReferenceFieldSubType, Row, BBRequest, + SyncAppResponse, + CreateAppResponse, + FetchAppsResponse, + UpdateAppClientResponse, + RevertAppClientResponse, + DestroyAppResponse, + ImportToUpdateAppRequest, + ImportToUpdateAppResponse, + SetRevertableAppVersionRequest, + SetRevertableAppVersionResponse, } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import sdk from "../../sdk" @@ -166,7 +176,7 @@ async function createInstance(appId: string, template: AppTemplate) { return { _id: appId } } -export const addSampleData = async (ctx: UserCtx) => { +export const addSampleData = async (ctx: UserCtx) => { const db = context.getAppDB() try { @@ -182,7 +192,7 @@ export const addSampleData = async (ctx: UserCtx) => { ctx.status = 200 } -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { ctx.body = await sdk.applications.fetch( ctx.query.status as AppStatus, ctx.user @@ -242,7 +252,9 @@ export async function fetchAppPackage( } } -async function performAppCreate(ctx: UserCtx) { +async function performAppCreate( + ctx: UserCtx +) { const apps = (await dbCore.getAllApps({ dev: true })) as App[] const { body } = ctx.request const { name, url, encryptionPassword, templateKey } = body @@ -510,7 +522,9 @@ async function appPostCreate(ctx: UserCtx, app: App) { } } -export async function create(ctx: UserCtx) { +export async function create( + ctx: UserCtx +) { const newApplication = await quotas.addApp(() => performAppCreate(ctx)) await appPostCreate(ctx, newApplication) await cache.bustCache(cache.CacheKey.CHECKLIST) @@ -553,7 +567,9 @@ export async function update( }) } -export async function updateClient(ctx: UserCtx) { +export async function updateClient( + ctx: UserCtx +) { // Get current app version const application = await sdk.applications.metadata.get() const currentVersion = application.version @@ -581,7 +597,9 @@ export async function updateClient(ctx: UserCtx) { ctx.body = app } -export async function revertClient(ctx: UserCtx) { +export async function revertClient( + ctx: UserCtx +) { // Check app can be reverted const application = await sdk.applications.metadata.get() if (!application.revertableVersion) { @@ -668,7 +686,7 @@ async function postDestroyApp(ctx: UserCtx) { } } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { await preDestroyApp(ctx) const result = await destroyApp(ctx) await postDestroyApp(ctx) @@ -676,7 +694,7 @@ export async function destroy(ctx: UserCtx) { ctx.body = result } -export async function unpublish(ctx: UserCtx) { +export async function unpublish(ctx: UserCtx) { const prodAppId = dbCore.getProdAppID(ctx.params.appId) const dbExists = await dbCore.dbExists(prodAppId) @@ -692,7 +710,7 @@ export async function unpublish(ctx: UserCtx) { builderSocket?.emitAppUnpublish(ctx) } -export async function sync(ctx: UserCtx) { +export async function sync(ctx: UserCtx) { const appId = ctx.params.appId try { ctx.body = await sdk.applications.syncApp(appId) @@ -701,10 +719,12 @@ export async function sync(ctx: UserCtx) { } } -export async function importToApp(ctx: UserCtx) { +export async function importToApp( + ctx: UserCtx +) { const { appId } = ctx.params const appExport = ctx.request.files?.appExport - const password = ctx.request.body.encryptionPassword as string + const password = ctx.request.body.encryptionPassword if (!appExport) { ctx.throw(400, "Must supply app export to import") } @@ -811,7 +831,7 @@ export async function updateAppPackage( } export async function setRevertableVersion( - ctx: UserCtx<{ revertableVersion: string }, App> + ctx: UserCtx ) { if (!env.isDev()) { ctx.status = 403 diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 2cf3da3dda..6a56f7d468 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -7,7 +7,12 @@ import { enableCronTrigger, } from "../../../automations/utils" import { backups } from "@budibase/pro" -import { App, AppBackupTrigger } from "@budibase/types" +import { + App, + AppBackupTrigger, + PublishAppResponse, + UserCtx, +} from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" @@ -123,7 +128,9 @@ export async function deploymentProgress(ctx: any) { } } -export const publishApp = async function (ctx: any) { +export const publishApp = async function ( + ctx: UserCtx +) { let deployment = new Deployment() console.log("Deployment object created") deployment.setStatus(DeploymentStatus.PENDING) diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 37450acf1d..3bbd72c12b 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -113,7 +113,7 @@ export async function syncUsersToAllApps(userIds: string[]) { export async function syncApp( appId: string, opts?: { automationOnly?: boolean } -) { +): Promise<{ message: string }> { if (env.DISABLE_AUTO_PROD_APP_SYNC) { return { message: diff --git a/packages/types/src/api/web/analytics.ts b/packages/types/src/api/web/analytics.ts index 172aeb8dd4..2585964f94 100644 --- a/packages/types/src/api/web/analytics.ts +++ b/packages/types/src/api/web/analytics.ts @@ -3,6 +3,10 @@ export enum PingSource { APP = "app", } +export interface AnalyticsEnabledResponse { + enabled: boolean +} + export interface AnalyticsPingRequest { source: PingSource timezone: string diff --git a/packages/types/src/api/web/apikeys.ts b/packages/types/src/api/web/apikeys.ts new file mode 100644 index 0000000000..e46d0d4db5 --- /dev/null +++ b/packages/types/src/api/web/apikeys.ts @@ -0,0 +1,10 @@ +export type ApiKeyFetchResponse = Record + +export interface ApiKeyUpdateRequest { + value: string +} + +export interface ApiKeyUpdateResponse { + _id: string + _rev: string +} diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts index 57422ceabc..168f18ede3 100644 --- a/packages/types/src/api/web/application.ts +++ b/packages/types/src/api/web/application.ts @@ -1,6 +1,10 @@ import type { PlanType } from "../../sdk" import type { Layout, App, Screen } from "../../documents" +export interface SyncAppResponse { + message: string +} + export interface CreateAppRequest { name: string url?: string @@ -12,6 +16,8 @@ export interface CreateAppRequest { file?: { path: string } } +export interface CreateAppResponse extends App {} + export interface DuplicateAppRequest { name: string url?: string @@ -37,6 +43,8 @@ export interface FetchAppPackageResponse { hasLock: boolean } +export type FetchAppsResponse = App[] + export interface PublishResponse { _id: string status: string @@ -45,3 +53,20 @@ export interface PublishResponse { export interface UpdateAppRequest extends Partial {} export interface UpdateAppResponse extends App {} +export interface UpdateAppClientResponse extends App {} +export interface RevertAppClientResponse extends App {} + +export interface DestroyAppResponse { + ok: boolean +} + +export interface ImportToUpdateAppRequest { + encryptionPassword?: string +} +export interface ImportToUpdateAppResponse { + message: string +} + +export interface SetRevertableAppVersionRequest { + revertableVersion: string +} diff --git a/packages/types/src/api/web/deployment.ts b/packages/types/src/api/web/deployment.ts new file mode 100644 index 0000000000..a9a007ed8f --- /dev/null +++ b/packages/types/src/api/web/deployment.ts @@ -0,0 +1,3 @@ +import { DeploymentDoc } from "../../documents" + +export interface PublishAppResponse extends DeploymentDoc {} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 27d51ce1b7..4c62d97daf 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -16,3 +16,5 @@ export * from "./layout" export * from "./query" export * from "./role" export * from "./plugins" +export * from "./apikeys" +export * from "./deployment" diff --git a/packages/types/src/documents/app/deployment.ts b/packages/types/src/documents/app/deployment.ts new file mode 100644 index 0000000000..63d3aea5f3 --- /dev/null +++ b/packages/types/src/documents/app/deployment.ts @@ -0,0 +1,7 @@ +export interface DeploymentDoc { + _id: string + verification: any + status?: string + err?: any + appUrl?: string +} diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index bb94c3b4da..51c6889f14 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -18,3 +18,4 @@ export * from "./sqlite" export * from "./snippet" export * from "./rowAction" export * from "./theme" +export * from "./deployment" diff --git a/packages/types/src/documents/global/apikeys.ts b/packages/types/src/documents/global/apikeys.ts new file mode 100644 index 0000000000..4b46f1d341 --- /dev/null +++ b/packages/types/src/documents/global/apikeys.ts @@ -0,0 +1,5 @@ +import { Document } from "../../" + +export interface ApiKeyDoc extends Document { + apiKeys: Record +} diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index b728439dd6..7d2f5a767c 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -7,3 +7,4 @@ export * from "./schedule" export * from "./templates" export * from "./environmentVariables" export * from "./auditLogs" +export * from "./apikeys" From b6a28bf56c0ece85c9614f6ac80b9d5db12bbb38 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 16:09:12 +0000 Subject: [PATCH 119/192] app user auth API. --- packages/server/src/api/controllers/auth.ts | 8 ++++---- packages/types/src/api/web/app/user.ts | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index 4ff592534d..6aab688c30 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -2,7 +2,7 @@ import { outputProcessing } from "../../utilities/rowProcessor" import { InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { roles, context, db as dbCore } from "@budibase/backend-core" -import { ContextUser, Row, UserCtx } from "@budibase/types" +import { AppSelfResponse, ContextUser, Row, UserCtx } from "@budibase/types" import sdk from "../../sdk" import { processUser } from "../../utilities/global" @@ -17,7 +17,7 @@ const addSessionAttributesToUser = (ctx: any) => { } } -export async function fetchSelf(ctx: UserCtx) { +export async function fetchSelf(ctx: UserCtx) { let userId = ctx.user.userId || ctx.user._id /* istanbul ignore next */ if (!userId || !ctx.isAuthenticated) { @@ -45,9 +45,9 @@ export async function fetchSelf(ctx: UserCtx) { try { const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA) // specifically needs to make sure is enriched - ctx.body = await outputProcessing(userTable, user as Row) + ctx.body = (await outputProcessing(userTable, user as Row)) as ContextUser } catch (err: any) { - let response + let response: ContextUser | {} // user didn't exist in app, don't pretend they do if (user.roleId === PUBLIC_ROLE) { response = {} diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts index 7faec83e9c..f5f2049724 100644 --- a/packages/types/src/api/web/app/user.ts +++ b/packages/types/src/api/web/app/user.ts @@ -7,3 +7,5 @@ export interface SetFlagRequest { flag: string value: any } + +export type AppSelfResponse = ContextUserMetadata | {} From b2b74aa048e82d7d17c5a60b00d42e754d766add Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 17:00:24 +0000 Subject: [PATCH 120/192] Automation API typing. --- .../server/src/api/controllers/automation.ts | 54 +++++++++++++++---- packages/server/src/automations/actions.ts | 4 +- packages/server/src/automations/triggers.ts | 2 +- packages/server/src/automations/utils.ts | 18 ++++--- packages/types/src/api/web/app/automation.ts | 21 ++++++++ packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/automation.ts | 52 +++++++++++++++++- .../src/documents/app/automation/schema.ts | 2 + 8 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 packages/types/src/api/web/app/automation.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index d8bc9d6b21..89f664853a 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -13,6 +13,22 @@ import { UserCtx, DeleteAutomationResponse, FetchAutomationResponse, + GetAutomationTriggerDefinitionsResponse, + GetAutomationStepDefinitionsResponse, + GetAutomationActionDefinitionsResponse, + FindAutomationResponse, + UpdateAutomationRequest, + UpdateAutomationResponse, + CreateAutomationRequest, + CreateAutomationResponse, + SearchAutomationLogsRequest, + SearchAutomationLogsResponse, + ClearAutomationLogRequest, + ClearAutomationLogResponse, + TriggerAutomationRequest, + TriggerAutomationResponse, + TestAutomationRequest, + TestAutomationResponse, } from "@budibase/types" import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" @@ -34,7 +50,7 @@ function getTriggerDefinitions() { *************************/ export async function create( - ctx: UserCtx + ctx: UserCtx ) { let automation = ctx.request.body automation.appId = ctx.appId @@ -55,7 +71,9 @@ export async function create( builderSocket?.emitAutomationUpdate(ctx, automation) } -export async function update(ctx: UserCtx) { +export async function update( + ctx: UserCtx +) { let automation = ctx.request.body automation.appId = ctx.appId @@ -80,7 +98,7 @@ export async function fetch(ctx: UserCtx) { ctx.body = { automations } } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { ctx.body = await sdk.automations.get(ctx.params.id) } @@ -96,11 +114,15 @@ export async function destroy(ctx: UserCtx) { builderSocket?.emitAutomationDeletion(ctx, automationId) } -export async function logSearch(ctx: UserCtx) { +export async function logSearch( + ctx: UserCtx +) { ctx.body = await automations.logs.logSearch(ctx.request.body) } -export async function clearLogError(ctx: UserCtx) { +export async function clearLogError( + ctx: UserCtx +) { const { automationId, appId } = ctx.request.body await context.doInAppContext(appId, async () => { const db = context.getProdAppDB() @@ -119,15 +141,21 @@ export async function clearLogError(ctx: UserCtx) { }) } -export async function getActionList(ctx: UserCtx) { +export async function getActionList( + ctx: UserCtx +) { ctx.body = await getActionDefinitions() } -export async function getTriggerList(ctx: UserCtx) { +export async function getTriggerList( + ctx: UserCtx +) { ctx.body = getTriggerDefinitions() } -export async function getDefinitionList(ctx: UserCtx) { +export async function getDefinitionList( + ctx: UserCtx +) { ctx.body = { trigger: getTriggerDefinitions(), action: await getActionDefinitions(), @@ -140,7 +168,9 @@ export async function getDefinitionList(ctx: UserCtx) { * * *********************/ -export async function trigger(ctx: UserCtx) { +export async function trigger( + ctx: UserCtx +) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) @@ -185,7 +215,7 @@ export async function trigger(ctx: UserCtx) { } } -function prepareTestInput(input: any) { +function prepareTestInput(input: TestAutomationRequest) { // prepare the test parameters if (input.id && input.row) { input.row._id = input.id @@ -196,7 +226,9 @@ function prepareTestInput(input: any) { return input } -export async function test(ctx: UserCtx) { +export async function test( + ctx: UserCtx +) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) await setTestFlag(automation._id!) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a8b69fa7d7..e91a708697 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -98,7 +98,9 @@ if (env.SELF_HOSTED) { BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition } -export async function getActionDefinitions() { +export async function getActionDefinitions(): Promise< + Record +> { if (await features.flags.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) { BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 70fda1f237..f422c14acb 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -148,7 +148,7 @@ export async function externalTrigger( user?: UserBindings }, { getResponses }: { getResponses?: boolean } = {} -): Promise { +): Promise> { if (automation.disabled) { throw new Error("Automation is disabled") } diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 365dc36b68..3eeeae5734 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -9,9 +9,11 @@ import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" import { Automation, + AutomationActionStepId, AutomationJob, AutomationStepDefinition, AutomationTriggerDefinition, + AutomationTriggerStepId, } from "@budibase/types" import { automationsEnabled } from "../features" import { helpers, REBOOT_CRON } from "@budibase/shared-core" @@ -120,19 +122,21 @@ export async function updateTestHistory( ) } -export function removeDeprecated( - definitions: Record< +export function removeDeprecated< + T extends + | Record + | Record +>(definitions: T): T { + const base: Record< string, - AutomationStepDefinition | AutomationTriggerDefinition - > -) { - const base = cloneDeep(definitions) + AutomationTriggerDefinition | AutomationStepDefinition + > = cloneDeep(definitions) for (let key of Object.keys(base)) { if (base[key].deprecated) { delete base[key] } } - return base + return base as T } // end the repetition and the job itself diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts new file mode 100644 index 0000000000..c10f19c88d --- /dev/null +++ b/packages/types/src/api/web/app/automation.ts @@ -0,0 +1,21 @@ +import { + AutomationActionStepId, + AutomationStepDefinition, + AutomationTriggerDefinition, + AutomationTriggerStepId, +} from "../../../documents" + +export type GetAutomationTriggerDefinitionsResponse = Record< + keyof typeof AutomationTriggerStepId, + AutomationTriggerDefinition +> + +export type GetAutomationActionDefinitionsResponse = Record< + keyof typeof AutomationActionStepId, + AutomationStepDefinition +> + +export interface GetAutomationStepDefinitionsResponse { + trigger: GetAutomationTriggerDefinitionsResponse + action: GetAutomationActionDefinitionsResponse +} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 55e1428fb9..3b7b183cc3 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -8,3 +8,4 @@ export * from "./permission" export * from "./attachment" export * from "./user" export * from "./rowAction" +export * from "./automation" diff --git a/packages/types/src/api/web/automation.ts b/packages/types/src/api/web/automation.ts index 06080fc667..0f0699939e 100644 --- a/packages/types/src/api/web/automation.ts +++ b/packages/types/src/api/web/automation.ts @@ -1,8 +1,58 @@ import { DocumentDestroyResponse } from "@budibase/nano" -import { Automation } from "../../documents" +import { + Automation, + AutomationLogPage, + AutomationStatus, + Row, +} from "../../documents" export interface DeleteAutomationResponse extends DocumentDestroyResponse {} export interface FetchAutomationResponse { automations: Automation[] } + +export interface FindAutomationResponse extends Automation {} + +export interface UpdateAutomationRequest extends Automation {} +export interface UpdateAutomationResponse { + message: string + automation: Automation +} + +export interface CreateAutomationRequest extends Automation {} +export interface CreateAutomationResponse { + message: string + automation: Automation +} + +export interface SearchAutomationLogsRequest { + startDate?: string + status?: AutomationStatus + automationId?: string + page?: string +} +export interface SearchAutomationLogsResponse extends AutomationLogPage {} + +export interface ClearAutomationLogRequest { + automationId: string + appId: string +} +export interface ClearAutomationLogResponse { + message: string +} + +export interface TriggerAutomationRequest { + fields: Record + // time in seconds + timeout: number +} +export type TriggerAutomationResponse = Record | undefined + +export interface TestAutomationRequest { + id?: string + revision?: string + fields: Record + row?: Row +} +export interface TestAutomationResponse {} diff --git a/packages/types/src/documents/app/automation/schema.ts b/packages/types/src/documents/app/automation/schema.ts index b8a19b7b45..efdf60a4e2 100644 --- a/packages/types/src/documents/app/automation/schema.ts +++ b/packages/types/src/documents/app/automation/schema.ts @@ -311,6 +311,7 @@ export type AutomationStep = type EmptyInputs = {} export type AutomationStepDefinition = Omit & { inputs: EmptyInputs + deprecated?: boolean } export type AutomationTriggerDefinition = Omit< @@ -318,6 +319,7 @@ export type AutomationTriggerDefinition = Omit< "id" | "inputs" > & { inputs: EmptyInputs + deprecated?: boolean } export type AutomationTriggerInputs = From d83bd47bd78b348044bba5b177b270f1c4c3015a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 17:13:39 +0000 Subject: [PATCH 121/192] Fixing some automation typing issues. --- .../server/src/api/controllers/automation.ts | 6 +++++- packages/server/src/automations/triggers.ts | 11 ++++++++++- packages/server/src/definitions/automations.ts | 3 +++ packages/server/src/threads/automation.ts | 18 ++++++++++++------ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 89f664853a..d13b0d3191 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -177,7 +177,7 @@ export async function trigger( let hasCollectStep = sdk.automations.utils.checkForCollectStep(automation) if (hasCollectStep && (await features.isSyncAutomationsEnabled())) { try { - const response: AutomationResults = await triggers.externalTrigger( + const response = await triggers.externalTrigger( automation, { fields: ctx.request.body.fields, @@ -188,6 +188,10 @@ export async function trigger( { getResponses: true } ) + if (!("steps" in response)) { + ctx.throw(400, "Unable to collect response") + } + let collectedValue = response.steps.find( step => step.stepId === AutomationActionStepId.COLLECT ) diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index f422c14acb..3a1e478522 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -20,6 +20,7 @@ import { AutomationStatus, AutomationRowEvent, UserBindings, + AutomationResults, } from "@budibase/types" import { executeInThread } from "../threads/automation" import { dataFilters, sdk } from "@budibase/shared-core" @@ -32,6 +33,14 @@ const JOB_OPTS = { import * as automationUtils from "../automations/automationUtils" import { doesTableExist } from "../sdk/app/tables/getters" +type DidNotTriggerResponse = { + outputs: { + success: false + status: AutomationStatus.STOPPED + } + message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET +} + async function getAllAutomations() { const db = context.getAppDB() let automations = await db.allDocs( @@ -148,7 +157,7 @@ export async function externalTrigger( user?: UserBindings }, { getResponses }: { getResponses?: boolean } = {} -): Promise> { +): Promise { if (automation.disabled) { throw new Error("Automation is disabled") } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index e45c555201..67d6e04e9d 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -26,3 +26,6 @@ export interface AutomationContext extends AutomationResults { company?: string } } + +export interface AutomationResponse + extends Omit {} diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 7e8f947580..2d10f5d1fb 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -30,7 +30,11 @@ import { UserBindings, isBasicSearchOperator, } from "@budibase/types" -import { AutomationContext, TriggerOutput } from "../definitions/automations" +import { + AutomationContext, + AutomationResponse, + TriggerOutput, +} from "../definitions/automations" import { WorkerCallback } from "./definitions" import { context, logging, configs } from "@budibase/backend-core" import { @@ -81,7 +85,7 @@ class Orchestrator { private job: Job private loopStepOutputs: LoopStep[] private stopped: boolean - private executionOutput: Omit + private executionOutput: AutomationResponse private currentUser: UserBindings | undefined constructor(job: AutomationJob) { @@ -257,7 +261,7 @@ class Orchestrator { }) } - async execute(): Promise { + async execute(): Promise { return tracer.trace( "Orchestrator.execute", { resource: "automation" }, @@ -723,7 +727,9 @@ export function execute(job: Job, callback: WorkerCallback) { }) } -export async function executeInThread(job: Job) { +export async function executeInThread( + job: Job +): Promise { const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") @@ -735,7 +741,7 @@ export async function executeInThread(job: Job) { }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT) }) - return await context.doInAppContext(appId, async () => { + return (await context.doInAppContext(appId, async () => { await context.ensureSnippetContext() const envVars = await sdkUtils.getEnvironmentVariables() // put into automation thread for whole context @@ -746,7 +752,7 @@ export async function executeInThread(job: Job) { timeoutPromise, ]) }) - }) + })) as AutomationResponse } export const removeStalled = async (job: Job) => { From 1094b880426c0ae3dbb269ff9de359e2847b23ab Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 17:25:08 +0000 Subject: [PATCH 122/192] Backup, component and datasource API typing. --- packages/server/src/api/controllers/backup.ts | 16 +++++++++------- packages/server/src/api/controllers/component.ts | 11 +++++++++-- .../server/src/api/controllers/datasource.ts | 14 ++++++++++---- packages/types/src/api/web/app/component.ts | 4 ++++ packages/types/src/api/web/app/datasource.ts | 11 +++++++++++ packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/application.ts | 8 ++++++++ 7 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 packages/types/src/api/web/app/component.ts diff --git a/packages/server/src/api/controllers/backup.ts b/packages/server/src/api/controllers/backup.ts index 8e1881a5fc..2104196139 100644 --- a/packages/server/src/api/controllers/backup.ts +++ b/packages/server/src/api/controllers/backup.ts @@ -1,14 +1,16 @@ import sdk from "../../sdk" import { events, context, db } from "@budibase/backend-core" import { DocumentType } from "../../db/utils" -import { App, Ctx } from "@budibase/types" +import { + App, + Ctx, + ExportAppDumpRequest, + ExportAppDumpResponse, +} from "@budibase/types" -interface ExportAppDumpRequest { - excludeRows: boolean - encryptPassword?: string -} - -export async function exportAppDump(ctx: Ctx) { +export async function exportAppDump( + ctx: Ctx +) { const { appId } = ctx.query as any const { excludeRows, encryptPassword } = ctx.request.body diff --git a/packages/server/src/api/controllers/component.ts b/packages/server/src/api/controllers/component.ts index 6d4d3e2d21..c6d8551adc 100644 --- a/packages/server/src/api/controllers/component.ts +++ b/packages/server/src/api/controllers/component.ts @@ -1,9 +1,16 @@ import { DocumentType } from "../../db/utils" -import { App, Plugin, UserCtx } from "@budibase/types" +import { + App, + FetchComponentDefinitionResponse, + Plugin, + UserCtx, +} from "@budibase/types" import { db as dbCore, context, tenancy } from "@budibase/backend-core" import { getComponentLibraryManifest } from "../../utilities/fileSystem" -export async function fetchAppComponentDefinitions(ctx: UserCtx) { +export async function fetchAppComponentDefinitions( + ctx: UserCtx +) { try { const db = context.getAppDB() const app = await db.get(DocumentType.APP_METADATA) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 97cf8db299..3b635a6038 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -23,13 +23,17 @@ import { Table, RowValue, DynamicVariable, + FetchDatasourcesResponse, + FindDatasourcesResponse, + DestroyDatasourceResponse, + FetchExternalSchemaResponse, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" import { isEqual } from "lodash" import { processTable } from "../../sdk/app/tables/getters" -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() } @@ -260,7 +264,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { } } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId @@ -291,12 +295,14 @@ export async function destroy(ctx: UserCtx) { builderSocket?.emitDatasourceDeletion(ctx, datasourceId) } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) ctx.body = await sdk.datasources.removeSecretSingle(datasource) } -export async function getExternalSchema(ctx: UserCtx) { +export async function getExternalSchema( + ctx: UserCtx +) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( datasource diff --git a/packages/types/src/api/web/app/component.ts b/packages/types/src/api/web/app/component.ts new file mode 100644 index 0000000000..486809a6b1 --- /dev/null +++ b/packages/types/src/api/web/app/component.ts @@ -0,0 +1,4 @@ +export type FetchComponentDefinitionResponse = Record< + string, + Record +> diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index f931665917..feb930b184 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -42,3 +42,14 @@ export interface BuildSchemaFromSourceResponse { datasource: Datasource errors: Record } + +export type FetchDatasourcesResponse = Datasource[] +export type FindDatasourcesResponse = Datasource + +export interface DestroyDatasourceResponse { + message: string +} + +export interface FetchExternalSchemaResponse { + schema: string +} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 3b7b183cc3..9cc0bf36b6 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -9,3 +9,4 @@ export * from "./attachment" export * from "./user" export * from "./rowAction" export * from "./automation" +export * from "./component" diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts index 168f18ede3..acbfedf5e0 100644 --- a/packages/types/src/api/web/application.ts +++ b/packages/types/src/api/web/application.ts @@ -1,5 +1,6 @@ import type { PlanType } from "../../sdk" import type { Layout, App, Screen } from "../../documents" +import { ReadStream } from "fs" export interface SyncAppResponse { message: string @@ -70,3 +71,10 @@ export interface ImportToUpdateAppResponse { export interface SetRevertableAppVersionRequest { revertableVersion: string } + +export interface ExportAppDumpRequest { + excludeRows: boolean + encryptPassword?: string +} + +export type ExportAppDumpResponse = ReadStream From fd71afde1dd46dd7e43c2b84fbe6e18e46361e87 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 29 Nov 2024 17:59:08 +0000 Subject: [PATCH 123/192] Deployment API typing. --- .../src/api/controllers/deploy/Deployment.ts | 5 ++-- .../src/api/controllers/deploy/index.ts | 29 ++++++++++++------- packages/types/src/api/web/deployment.ts | 13 ++++++++- .../types/src/documents/app/deployment.ts | 17 ++++++++++- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/Deployment.ts b/packages/server/src/api/controllers/deploy/Deployment.ts index fe817730b6..9d5e543187 100644 --- a/packages/server/src/api/controllers/deploy/Deployment.ts +++ b/packages/server/src/api/controllers/deploy/Deployment.ts @@ -1,4 +1,5 @@ import { context, utils } from "@budibase/backend-core" +import { DeploymentStatus } from "@budibase/types" /** * This is used to pass around information about the deployment that is occurring @@ -6,7 +7,7 @@ import { context, utils } from "@budibase/backend-core" export default class Deployment { _id: string verification: any - status?: string + status?: DeploymentStatus err?: any appUrl?: string @@ -25,7 +26,7 @@ export default class Deployment { return this.verification } - setStatus(status: string, err?: any) { + setStatus(status: DeploymentStatus, err?: any) { this.status = status if (err) { this.err = err diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 6a56f7d468..652086f571 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -10,22 +10,23 @@ import { backups } from "@budibase/pro" import { App, AppBackupTrigger, + DeploymentDoc, + FetchDeploymentResponse, PublishAppResponse, UserCtx, + DeploymentStatus, + DeploymentProgressResponse, } from "@budibase/types" import sdk from "../../../sdk" import { builderSocket } from "../../../websockets" // the max time we can wait for an invalidation to complete before considering it failed const MAX_PENDING_TIME_MS = 30 * 60000 -const DeploymentStatus = { - SUCCESS: "SUCCESS", - PENDING: "PENDING", - FAILURE: "FAILURE", -} // checks that deployments are in a good state, any pending will be updated -async function checkAllDeployments(deployments: any) { +async function checkAllDeployments( + deployments: any +): Promise<{ updated: boolean; deployments: DeploymentDoc }> { let updated = false let deployment: any for (deployment of Object.values(deployments.history)) { @@ -101,7 +102,9 @@ async function initDeployedApp(prodAppId: any) { }) } -export async function fetchDeployments(ctx: any) { +export async function fetchDeployments( + ctx: UserCtx +) { try { const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) @@ -109,17 +112,21 @@ export async function fetchDeployments(ctx: any) { if (updated) { await db.put(deployments) } - ctx.body = Object.values(deployments.history).reverse() + ctx.body = deployments.history + ? Object.values(deployments.history).reverse() + : [] } catch (err) { ctx.body = [] } } -export async function deploymentProgress(ctx: any) { +export async function deploymentProgress( + ctx: UserCtx +) { try { const db = context.getAppDB() - const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) - ctx.body = deploymentDoc[ctx.params.deploymentId] + const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) + ctx.body = deploymentDoc.history?.[ctx.params.deploymentId] } catch (err) { ctx.throw( 500, diff --git a/packages/types/src/api/web/deployment.ts b/packages/types/src/api/web/deployment.ts index a9a007ed8f..9e4caccc24 100644 --- a/packages/types/src/api/web/deployment.ts +++ b/packages/types/src/api/web/deployment.ts @@ -1,3 +1,14 @@ -import { DeploymentDoc } from "../../documents" +import { DeploymentDoc, DeploymentStatus } from "../../documents" export interface PublishAppResponse extends DeploymentDoc {} + +export type DeploymentProgressResponse = + | { + _id: string + appId: string + status?: DeploymentStatus + updatedAt: number + } + | undefined + +export type FetchDeploymentResponse = DeploymentProgressResponse[] diff --git a/packages/types/src/documents/app/deployment.ts b/packages/types/src/documents/app/deployment.ts index 63d3aea5f3..3216ebae3d 100644 --- a/packages/types/src/documents/app/deployment.ts +++ b/packages/types/src/documents/app/deployment.ts @@ -1,7 +1,22 @@ +export enum DeploymentStatus { + SUCCESS = "SUCCESS", + PENDING = "PENDING", + FAILURE = "FAILURE", +} + export interface DeploymentDoc { _id: string verification: any - status?: string + status?: DeploymentStatus + history?: Record< + string, + { + _id: string + appId: string + status?: DeploymentStatus + updatedAt: number + } + > err?: any appUrl?: string } From ffca69ea1c3385ae7e67b726504422e04795cb2a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 2 Dec 2024 11:38:11 +0000 Subject: [PATCH 124/192] Bump version to 3.2.17 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 74e0cde617..489986b57f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.16", + "version": "3.2.17", "npmClient": "yarn", "concurrency": 20, "command": { From b1cd134ee04104b436080232ea963b2646c6bda1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 14:42:46 +0000 Subject: [PATCH 125/192] Dev api. --- packages/server/src/api/controllers/application.ts | 1 - packages/server/src/api/controllers/dev.ts | 4 ++-- packages/types/src/api/web/dev.ts | 3 +++ packages/types/src/api/web/index.ts | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 packages/types/src/api/web/dev.ts diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 9170ba54c6..3925df8ab0 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -68,7 +68,6 @@ import { ImportToUpdateAppRequest, ImportToUpdateAppResponse, SetRevertableAppVersionRequest, - SetRevertableAppVersionResponse, } from "@budibase/types" import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts" import sdk from "../../sdk" diff --git a/packages/server/src/api/controllers/dev.ts b/packages/server/src/api/controllers/dev.ts index 497da088c6..41e4a3a20a 100644 --- a/packages/server/src/api/controllers/dev.ts +++ b/packages/server/src/api/controllers/dev.ts @@ -11,7 +11,7 @@ import { db as dbCore, cache, } from "@budibase/backend-core" -import { App } from "@budibase/types" +import { App, Ctx, GetVersionResponse } from "@budibase/types" async function redirect( ctx: any, @@ -131,7 +131,7 @@ export async function revert(ctx: any) { } } -export async function getBudibaseVersion(ctx: any) { +export async function getBudibaseVersion(ctx: Ctx) { const version = envCore.VERSION ctx.body = { version, diff --git a/packages/types/src/api/web/dev.ts b/packages/types/src/api/web/dev.ts new file mode 100644 index 0000000000..f3195f58d3 --- /dev/null +++ b/packages/types/src/api/web/dev.ts @@ -0,0 +1,3 @@ +export interface GetVersionResponse { + version: string +} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 4c62d97daf..4021eafee8 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -18,3 +18,4 @@ export * from "./role" export * from "./plugins" export * from "./apikeys" export * from "./deployment" +export * from "./dev" From bc3658e2cb7452a4dd78c7b6d01dac172436af2d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 15:12:27 +0000 Subject: [PATCH 126/192] Adding fix for build issue. --- packages/server/src/api/controllers/automation.ts | 1 - packages/server/src/automations/tests/loop.spec.ts | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index d13b0d3191..c843dca89b 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -9,7 +9,6 @@ import { App, Automation, AutomationActionStepId, - AutomationResults, UserCtx, DeleteAutomationResponse, FetchAutomationResponse, diff --git a/packages/server/src/automations/tests/loop.spec.ts b/packages/server/src/automations/tests/loop.spec.ts index 372c3855b3..2199a2a3a0 100644 --- a/packages/server/src/automations/tests/loop.spec.ts +++ b/packages/server/src/automations/tests/loop.spec.ts @@ -3,7 +3,7 @@ import * as triggers from "../triggers" import { loopAutomation } from "../../tests/utilities/structures" import { context } from "@budibase/backend-core" import * as setup from "./utilities" -import { Table, LoopStepType } from "@budibase/types" +import { Table, LoopStepType, AutomationResults } from "@budibase/types" import * as loopUtils from "../loopUtils" import { LoopInput } from "../../definitions/automations" @@ -20,15 +20,19 @@ describe("Attempt to run a basic loop automation", () => { afterAll(setup.afterAll) - async function runLoop(loopOpts?: LoopInput) { + async function runLoop(loopOpts?: LoopInput): Promise { const appId = config.getAppId() return await context.doInAppContext(appId, async () => { const params = { fields: { appId } } - return await triggers.externalTrigger( + const result = await triggers.externalTrigger( loopAutomation(table._id!, loopOpts), params, { getResponses: true } ) + if ("outputs" in result && !result.outputs.success) { + throw new Error("Unable to proceed - failed to return anything.") + } + return result as AutomationResults }) } From fe9e4d1c5a8768341f1d23a763adeece3b850b1e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 15:26:12 +0000 Subject: [PATCH 127/192] More typing fixes. --- packages/server/src/api/controllers/webhook.ts | 15 ++++++++++----- .../src/automations/steps/triggerAutomationRun.ts | 13 ++++++++----- packages/server/src/automations/triggers.ts | 8 ++++++++ .../src/tests/utilities/TestConfiguration.ts | 2 +- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/webhook.ts b/packages/server/src/api/controllers/webhook.ts index d667134f4d..0f8f7be0ef 100644 --- a/packages/server/src/api/controllers/webhook.ts +++ b/packages/server/src/api/controllers/webhook.ts @@ -10,6 +10,7 @@ import { } from "@budibase/types" import sdk from "../../sdk" import * as pro from "@budibase/pro" +import { isAutomationResults } from "../../automations/triggers" const toJsonSchema = require("to-json-schema") const validate = require("jsonschema").validate @@ -94,12 +95,16 @@ export async function trigger(ctx: BBContext) { { getResponses: true } ) - let collectedValue = response.steps.find( - (step: any) => step.stepId === AutomationActionStepId.COLLECT - ) + if (triggers.isAutomationResults(response)) { + let collectedValue = response.steps.find( + (step: any) => step.stepId === AutomationActionStepId.COLLECT + ) - ctx.status = 200 - ctx.body = collectedValue.outputs + ctx.status = 200 + ctx.body = collectedValue?.outputs + } else { + ctx.throw(400, "Automation did not have a collect block.") + } } else { await triggers.externalTrigger(target, { body: ctx.request.body, diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index c43f46b6f9..f1cf65b182 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -3,7 +3,6 @@ import { AutomationStepDefinition, AutomationStepType, AutomationIOType, - AutomationResults, Automation, AutomationCustomIOType, TriggerAutomationStepInputs, @@ -78,7 +77,7 @@ export async function run({ const db = context.getAppDB() let automation = await db.get(inputs.automation.automationId) - const response: AutomationResults = await triggers.externalTrigger( + const response = await triggers.externalTrigger( automation, { fields: { ...fieldParams }, @@ -88,9 +87,13 @@ export async function run({ { getResponses: true } ) - return { - success: true, - value: response.steps, + if (triggers.isAutomationResults(response)) { + return { + success: true, + value: response.steps, + } + } else { + throw new Error("Automation did not have a collect block") } } } else { diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 3a1e478522..ed0aaaf3ec 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -148,6 +148,14 @@ function rowPassesFilters(row: Row, filters: SearchFilters) { return filteredRows.length > 0 } +export function isAutomationResults( + response: AutomationResults | DidNotTriggerResponse | AutomationJob +): response is AutomationResults { + return ( + response !== null && "steps" in response && Array.isArray(response.steps) + ) +} + export async function externalTrigger( automation: Automation, params: { diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index abecf6df44..2d36e7855b 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -621,7 +621,7 @@ export default class TestConfiguration { } async unpublish() { - const response = await this._req(appController.unpublish, { + const response = await this._req(appController.unpublish, undefined, { appId: this.appId, }) this.prodAppId = undefined From 24be6ef496376a88532e9d92c4fd20d320f60f6d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 15:32:59 +0000 Subject: [PATCH 128/192] Linting. --- packages/server/src/api/controllers/webhook.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/webhook.ts b/packages/server/src/api/controllers/webhook.ts index 0f8f7be0ef..7c648ea827 100644 --- a/packages/server/src/api/controllers/webhook.ts +++ b/packages/server/src/api/controllers/webhook.ts @@ -10,7 +10,6 @@ import { } from "@budibase/types" import sdk from "../../sdk" import * as pro from "@budibase/pro" -import { isAutomationResults } from "../../automations/triggers" const toJsonSchema = require("to-json-schema") const validate = require("jsonschema").validate From e778022a9842d4878322dc9b7ecfbdd1909d8514 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 16:17:57 +0000 Subject: [PATCH 129/192] Enrich the datasource when not passing the datasource ID. --- packages/server/src/sdk/app/rows/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 50170dd8bd..c19654d817 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -96,7 +96,7 @@ export async function enrichQueryJson( }) } } else { - datasource = json.endpoint.datasourceId + datasource = await sdk.datasources.enrich(json.endpoint.datasourceId) } let tables: Record From 05ba3230fad6ba5fb409aae5def21d248dd0ee38 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 2 Dec 2024 16:30:23 +0000 Subject: [PATCH 130/192] Bump version to 3.2.18 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 489986b57f..8eb8cf46a1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.17", + "version": "3.2.18", "npmClient": "yarn", "concurrency": 20, "command": { From 98de7e69d1e5dd8d367191dec4117b46697fd1e6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 16:40:50 +0000 Subject: [PATCH 131/192] Last parts of dev API. --- packages/server/src/api/controllers/dev.ts | 12 +++++++++--- packages/types/src/api/web/dev.ts | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/dev.ts b/packages/server/src/api/controllers/dev.ts index 41e4a3a20a..ad0909a294 100644 --- a/packages/server/src/api/controllers/dev.ts +++ b/packages/server/src/api/controllers/dev.ts @@ -11,7 +11,13 @@ import { db as dbCore, cache, } from "@budibase/backend-core" -import { App, Ctx, GetVersionResponse } from "@budibase/types" +import { + App, + ClearDevLockResponse, + Ctx, + GetVersionResponse, + RevertAppResponse, +} from "@budibase/types" async function redirect( ctx: any, @@ -69,7 +75,7 @@ export function buildRedirectDelete(path: string) { } } -export async function clearLock(ctx: any) { +export async function clearLock(ctx: Ctx) { const { appId } = ctx.params try { await redisClearLock(appId, ctx.user) @@ -81,7 +87,7 @@ export async function clearLock(ctx: any) { } } -export async function revert(ctx: any) { +export async function revert(ctx: Ctx) { const { appId } = ctx.params const productionAppId = dbCore.getProdAppID(appId) diff --git a/packages/types/src/api/web/dev.ts b/packages/types/src/api/web/dev.ts index f3195f58d3..461676a480 100644 --- a/packages/types/src/api/web/dev.ts +++ b/packages/types/src/api/web/dev.ts @@ -1,3 +1,11 @@ export interface GetVersionResponse { version: string } + +export interface ClearDevLockResponse { + message: string +} + +export interface RevertAppResponse { + message: string +} From bb4d7e0b3270a908327f72b3d6f3488c4d2cdc71 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 16:48:34 +0000 Subject: [PATCH 132/192] Integration API. --- .../server/src/api/controllers/integration.ts | 17 +++++++++++++---- packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/app/integration.ts | 8 ++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 packages/types/src/api/web/app/integration.ts diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 57038f8401..9f770ad049 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -1,12 +1,17 @@ import { getDefinition, getDefinitions } from "../../integrations" -import { SourceName, UserCtx } from "@budibase/types" +import { + SourceName, + UserCtx, + FetchIntegrationsResponse, + FindIntegrationResponse, +} from "@budibase/types" const DISABLED_EXTERNAL_INTEGRATIONS = [ SourceName.AIRTABLE, SourceName.BUDIBASE, ] -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { const definitions = await getDefinitions() for (let disabledIntegration of DISABLED_EXTERNAL_INTEGRATIONS) { delete definitions[disabledIntegration] @@ -14,10 +19,14 @@ export async function fetch(ctx: UserCtx) { ctx.body = definitions } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { const sourceType = ctx.params?.type if (DISABLED_EXTERNAL_INTEGRATIONS.indexOf(sourceType) !== -1) { ctx.throw(400, `Invalid source type - ${sourceType} is not supported.`) } - ctx.body = await getDefinition(ctx.params.type) + const integration = await getDefinition(ctx.params.type) + if (!integration) { + ctx.throw(400, "Integration not found") + } + ctx.body = integration } diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 9cc0bf36b6..614446a1b7 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -10,3 +10,4 @@ export * from "./user" export * from "./rowAction" export * from "./automation" export * from "./component" +export * from "./integration" diff --git a/packages/types/src/api/web/app/integration.ts b/packages/types/src/api/web/app/integration.ts new file mode 100644 index 0000000000..8e52a32ba9 --- /dev/null +++ b/packages/types/src/api/web/app/integration.ts @@ -0,0 +1,8 @@ +import { Integration, SourceName } from "../../../sdk" + +export type FetchIntegrationsResponse = Record< + SourceName, + Integration | undefined +> + +export type FindIntegrationResponse = Integration From 82e4e190fccb506613e266940de4e52839f35adc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 16:53:00 +0000 Subject: [PATCH 133/192] Layout API. --- packages/server/src/api/controllers/layout.ts | 3 ++- packages/types/src/api/web/layout.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts index c0406f50ac..447b437a23 100644 --- a/packages/server/src/api/controllers/layout.ts +++ b/packages/server/src/api/controllers/layout.ts @@ -3,6 +3,7 @@ import { generateLayoutID, getScreenParams } from "../../db/utils" import { events, context } from "@budibase/backend-core" import { BBContext, + DeleteLayoutResponse, Layout, SaveLayoutRequest, SaveLayoutResponse, @@ -32,7 +33,7 @@ export async function save( ctx.status = 200 } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const layoutId = ctx.params.layoutId, layoutRev = ctx.params.layoutRev diff --git a/packages/types/src/api/web/layout.ts b/packages/types/src/api/web/layout.ts index 50512777ef..45f7fb9d0a 100644 --- a/packages/types/src/api/web/layout.ts +++ b/packages/types/src/api/web/layout.ts @@ -3,3 +3,7 @@ import { Layout } from "../../documents" export interface SaveLayoutRequest extends Layout {} export interface SaveLayoutResponse extends Layout {} + +export interface DeleteLayoutResponse { + message: string +} From d42c4118fe0c96a0bd6cf47ebed755938a8bca17 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 16:58:35 +0000 Subject: [PATCH 134/192] Cleanup. --- packages/server/src/api/controllers/apikeys.ts | 6 +++--- packages/server/src/api/controllers/application.ts | 4 ++-- packages/server/src/api/controllers/datasource.ts | 4 ++-- packages/types/src/api/web/apikeys.ts | 4 ++-- packages/types/src/api/web/app/datasource.ts | 2 +- packages/types/src/api/web/application.ts | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/apikeys.ts b/packages/server/src/api/controllers/apikeys.ts index 8f841cd457..95253b09c5 100644 --- a/packages/server/src/api/controllers/apikeys.ts +++ b/packages/server/src/api/controllers/apikeys.ts @@ -4,8 +4,8 @@ import { UserCtx, ApiKeyDoc, ApiKeyFetchResponse, - ApiKeyUpdateRequest, - ApiKeyUpdateResponse, + UpdateApiKeyRequest, + UpdateApiKeyResponse, } from "@budibase/types" const KEYS_DOC = dbCore.StaticDatabases.GLOBAL.docs.apiKeys @@ -40,7 +40,7 @@ export async function fetch(ctx: UserCtx) { } export async function update( - ctx: UserCtx + ctx: UserCtx ) { const key = ctx.params.key const value = ctx.request.body.value diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 3925df8ab0..d032f14150 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -64,7 +64,7 @@ import { FetchAppsResponse, UpdateAppClientResponse, RevertAppClientResponse, - DestroyAppResponse, + DeleteAppResponse, ImportToUpdateAppRequest, ImportToUpdateAppResponse, SetRevertableAppVersionRequest, @@ -685,7 +685,7 @@ async function postDestroyApp(ctx: UserCtx) { } } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { await preDestroyApp(ctx) const result = await destroyApp(ctx) await postDestroyApp(ctx) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 3b635a6038..c4492f304c 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -25,7 +25,7 @@ import { DynamicVariable, FetchDatasourcesResponse, FindDatasourcesResponse, - DestroyDatasourceResponse, + DeleteDatasourceResponse, FetchExternalSchemaResponse, } from "@budibase/types" import sdk from "../../sdk" @@ -264,7 +264,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { } } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId diff --git a/packages/types/src/api/web/apikeys.ts b/packages/types/src/api/web/apikeys.ts index e46d0d4db5..1d089cd0ac 100644 --- a/packages/types/src/api/web/apikeys.ts +++ b/packages/types/src/api/web/apikeys.ts @@ -1,10 +1,10 @@ export type ApiKeyFetchResponse = Record -export interface ApiKeyUpdateRequest { +export interface UpdateApiKeyRequest { value: string } -export interface ApiKeyUpdateResponse { +export interface UpdateApiKeyResponse { _id: string _rev: string } diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index feb930b184..6f982d7060 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -46,7 +46,7 @@ export interface BuildSchemaFromSourceResponse { export type FetchDatasourcesResponse = Datasource[] export type FindDatasourcesResponse = Datasource -export interface DestroyDatasourceResponse { +export interface DeleteDatasourceResponse { message: string } diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts index acbfedf5e0..ed999ee574 100644 --- a/packages/types/src/api/web/application.ts +++ b/packages/types/src/api/web/application.ts @@ -57,7 +57,7 @@ export interface UpdateAppResponse extends App {} export interface UpdateAppClientResponse extends App {} export interface RevertAppClientResponse extends App {} -export interface DestroyAppResponse { +export interface DeleteAppResponse { ok: boolean } From c073500d502643ee7227ce98802cd98685d75997 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 2 Dec 2024 17:42:46 +0000 Subject: [PATCH 135/192] Metadata API. --- .../server/src/api/controllers/metadata.ts | 37 ++++++++++--------- .../{metadata.spec.js => metadata.spec.ts} | 24 ++++++------ packages/server/src/automations/utils.ts | 4 +- packages/server/src/constants/index.ts | 5 --- .../server/src/sdk/app/automations/crud.ts | 11 ++---- packages/server/src/utilities/index.ts | 2 +- packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/app/metadata.ts | 14 +++++++ packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/app/metadata.ts | 4 ++ 10 files changed, 59 insertions(+), 44 deletions(-) rename packages/server/src/api/routes/tests/{metadata.spec.js => metadata.spec.ts} (69%) create mode 100644 packages/types/src/api/web/app/metadata.ts create mode 100644 packages/types/src/documents/app/metadata.ts diff --git a/packages/server/src/api/controllers/metadata.ts b/packages/server/src/api/controllers/metadata.ts index f579b14499..85c1ab2be8 100644 --- a/packages/server/src/api/controllers/metadata.ts +++ b/packages/server/src/api/controllers/metadata.ts @@ -1,24 +1,35 @@ -import { MetadataTypes } from "../../constants" import { generateMetadataID } from "../../db/utils" import { saveEntityMetadata, deleteEntityMetadata } from "../../utilities" import { context } from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { + UserCtx, + MetadataType, + GetMetadataTypesResponse, + SaveMetadataRequest, + SaveMetadataResponse, + DeleteMetadataResponse, + FindMetadataResponse, +} from "@budibase/types" -export async function getTypes(ctx: BBContext) { +export async function getTypes(ctx: UserCtx) { ctx.body = { - types: MetadataTypes, + types: MetadataType, } } -export async function saveMetadata(ctx: BBContext) { +export async function saveMetadata( + ctx: UserCtx +) { const { type, entityId } = ctx.params - if (type === MetadataTypes.AUTOMATION_TEST_HISTORY) { + if (type === MetadataType.AUTOMATION_TEST_HISTORY) { ctx.throw(400, "Cannot save automation history type") } ctx.body = await saveEntityMetadata(type, entityId, ctx.request.body) } -export async function deleteMetadata(ctx: BBContext) { +export async function deleteMetadata( + ctx: UserCtx +) { const { type, entityId } = ctx.params await deleteEntityMetadata(type, entityId) ctx.body = { @@ -26,17 +37,9 @@ export async function deleteMetadata(ctx: BBContext) { } } -export async function getMetadata(ctx: BBContext) { +export async function getMetadata(ctx: UserCtx) { const { type, entityId } = ctx.params const db = context.getAppDB() const id = generateMetadataID(type, entityId) - try { - ctx.body = await db.get(id) - } catch (err: any) { - if (err.status === 404) { - ctx.body = {} - } else { - ctx.throw(err.status, err) - } - } + ctx.body = (await db.tryGet(id)) || {} } diff --git a/packages/server/src/api/routes/tests/metadata.spec.js b/packages/server/src/api/routes/tests/metadata.spec.ts similarity index 69% rename from packages/server/src/api/routes/tests/metadata.spec.js rename to packages/server/src/api/routes/tests/metadata.spec.ts index d5563f417e..6e991a272e 100644 --- a/packages/server/src/api/routes/tests/metadata.spec.js +++ b/packages/server/src/api/routes/tests/metadata.spec.ts @@ -1,11 +1,11 @@ -const { testAutomation } = require("./utilities/TestFunctions") -const setup = require("./utilities") -const { MetadataTypes } = require("../../../constants") +import { testAutomation } from "./utilities/TestFunctions" +import * as setup from "./utilities" +import { MetadataType, Automation } from "@budibase/types" describe("/metadata", () => { let request = setup.getRequest() let config = setup.getConfig() - let automation + let automation: Automation afterAll(setup.afterAll) @@ -15,8 +15,8 @@ describe("/metadata", () => { }) async function createMetadata( - data, - type = MetadataTypes.AUTOMATION_TEST_INPUT + data: Record, + type = MetadataType.AUTOMATION_TEST_INPUT ) { const res = await request .post(`/api/metadata/${type}/${automation._id}`) @@ -27,7 +27,7 @@ describe("/metadata", () => { expect(res.body._rev).toBeDefined() } - async function getMetadata(type) { + async function getMetadata(type: MetadataType) { const res = await request .get(`/api/metadata/${type}/${automation._id}`) .set(config.defaultHeaders()) @@ -39,14 +39,14 @@ describe("/metadata", () => { describe("save", () => { it("should be able to save some metadata", async () => { await createMetadata({ test: "a" }) - const testInput = await getMetadata(MetadataTypes.AUTOMATION_TEST_INPUT) + const testInput = await getMetadata(MetadataType.AUTOMATION_TEST_INPUT) expect(testInput.test).toBe("a") }) it("should save history metadata on automation run", async () => { // this should have created some history - await testAutomation(config, automation) - const metadata = await getMetadata(MetadataTypes.AUTOMATION_TEST_HISTORY) + await testAutomation(config, automation, {}) + const metadata = await getMetadata(MetadataType.AUTOMATION_TEST_HISTORY) expect(metadata).toBeDefined() expect(metadata.history.length).toBe(1) expect(typeof metadata.history[0].occurredAt).toBe("number") @@ -57,13 +57,13 @@ describe("/metadata", () => { it("should be able to delete some test inputs", async () => { const res = await request .delete( - `/api/metadata/${MetadataTypes.AUTOMATION_TEST_INPUT}/${automation._id}` + `/api/metadata/${MetadataType.AUTOMATION_TEST_INPUT}/${automation._id}` ) .set(config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) expect(res.body.message).toBeDefined() - const metadata = await getMetadata(MetadataTypes.AUTOMATION_TEST_INPUT) + const metadata = await getMetadata(MetadataType.AUTOMATION_TEST_INPUT) expect(metadata.test).toBeUndefined() }) }) diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 3eeeae5734..292b288a70 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -2,7 +2,6 @@ import { Thread, ThreadType } from "../threads" import { definitions } from "./triggerInfo" import { automationQueue } from "./bullboard" import { updateEntityMetadata } from "../utilities" -import { MetadataTypes } from "../constants" import { context, db as dbCore, utils } from "@budibase/backend-core" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" @@ -14,6 +13,7 @@ import { AutomationStepDefinition, AutomationTriggerDefinition, AutomationTriggerStepId, + MetadataType, } from "@budibase/types" import { automationsEnabled } from "../features" import { helpers, REBOOT_CRON } from "@budibase/shared-core" @@ -107,7 +107,7 @@ export async function updateTestHistory( history: any ) { return updateEntityMetadata( - MetadataTypes.AUTOMATION_TEST_HISTORY, + MetadataType.AUTOMATION_TEST_HISTORY, automation._id, (metadata: any) => { if (metadata && Array.isArray(metadata.history)) { diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 89e2f26516..fde1efd1b9 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -124,11 +124,6 @@ export enum BaseQueryVerbs { DELETE = "delete", } -export enum MetadataTypes { - AUTOMATION_TEST_INPUT = "automationTestInput", - AUTOMATION_TEST_HISTORY = "automationTestHistory", -} - export enum InvalidColumns { ID = "_id", REV = "_rev", diff --git a/packages/server/src/sdk/app/automations/crud.ts b/packages/server/src/sdk/app/automations/crud.ts index b982d5b45e..d16ae2d042 100644 --- a/packages/server/src/sdk/app/automations/crud.ts +++ b/packages/server/src/sdk/app/automations/crud.ts @@ -3,10 +3,10 @@ import { RequiredKeys, Webhook, WebhookActionType, + MetadataType, } from "@budibase/types" import { generateAutomationID, getAutomationParams } from "../../../db/utils" import { deleteEntityMetadata } from "../../../utilities" -import { MetadataTypes } from "../../../constants" import { context, events, @@ -161,7 +161,7 @@ export async function update(automation: Automation) { if (oldAutoTrigger && oldAutoTrigger.id !== newAutoTrigger?.id) { await events.automation.triggerUpdated(automation) await deleteEntityMetadata( - MetadataTypes.AUTOMATION_TEST_INPUT, + MetadataType.AUTOMATION_TEST_INPUT, automation._id! ) } @@ -183,11 +183,8 @@ export async function remove(automationId: string, rev: string) { }) // delete metadata first - await deleteEntityMetadata(MetadataTypes.AUTOMATION_TEST_INPUT, automationId) - await deleteEntityMetadata( - MetadataTypes.AUTOMATION_TEST_HISTORY, - automationId - ) + await deleteEntityMetadata(MetadataType.AUTOMATION_TEST_INPUT, automationId) + await deleteEntityMetadata(MetadataType.AUTOMATION_TEST_HISTORY, automationId) const result = await db.remove(automationId, rev) diff --git a/packages/server/src/utilities/index.ts b/packages/server/src/utilities/index.ts index 331a8e266f..db57b4ec12 100644 --- a/packages/server/src/utilities/index.ts +++ b/packages/server/src/utilities/index.ts @@ -88,7 +88,7 @@ export async function saveEntityMetadata( type: string, entityId: string, metadata: Document -) { +): Promise { return updateEntityMetadata(type, entityId, () => { return metadata }) diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 614446a1b7..40c8ebf9ca 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -11,3 +11,4 @@ export * from "./rowAction" export * from "./automation" export * from "./component" export * from "./integration" +export * from "./metadata" diff --git a/packages/types/src/api/web/app/metadata.ts b/packages/types/src/api/web/app/metadata.ts new file mode 100644 index 0000000000..5940f8e3d0 --- /dev/null +++ b/packages/types/src/api/web/app/metadata.ts @@ -0,0 +1,14 @@ +import { MetadataType, Document } from "../../../documents" + +export interface GetMetadataTypesResponse { + types: typeof MetadataType +} + +export interface SaveMetadataRequest extends Document {} +export interface SaveMetadataResponse extends Document {} + +export interface DeleteMetadataResponse { + message: string +} + +export interface FindMetadataResponse extends Document {} diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index 51c6889f14..d6c4fd8637 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -19,3 +19,4 @@ export * from "./snippet" export * from "./rowAction" export * from "./theme" export * from "./deployment" +export * from "./metadata" diff --git a/packages/types/src/documents/app/metadata.ts b/packages/types/src/documents/app/metadata.ts new file mode 100644 index 0000000000..421e8a2f0d --- /dev/null +++ b/packages/types/src/documents/app/metadata.ts @@ -0,0 +1,4 @@ +export enum MetadataType { + AUTOMATION_TEST_INPUT = "automationTestInput", + AUTOMATION_TEST_HISTORY = "automationTestHistory", +} From 7d0ae9d567ee58914c8f9e584c6205297d196add Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 3 Dec 2024 10:11:57 +0000 Subject: [PATCH 136/192] no need to add run func to definition --- packages/server/src/automations/actions.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index edac2ed690..0a80714032 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -98,7 +98,6 @@ if (env.SELF_HOSTED) { BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition if (env.isTest()) { - ACTION_IMPLS["OPENAI"] = openai.run BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition } } From 10dbb9fa7aa6bfae1b9edfa9b63567d8aa50898d Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 3 Dec 2024 10:33:18 +0000 Subject: [PATCH 137/192] re-add removed comments --- packages/server/src/automations/tests/openai.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index a4d98fc9d3..9f2bc50599 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -158,6 +158,8 @@ describe("test the openai action", () => { const llmInstance = mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) }) From 15991d2316723eb5da2ada5880fddcf75caa67ca Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 3 Dec 2024 11:01:17 +0000 Subject: [PATCH 138/192] Bump version to 3.2.19 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 8eb8cf46a1..814102c86a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.18", + "version": "3.2.19", "npmClient": "yarn", "concurrency": 20, "command": { From 1b15ef3b7e440fd474a6c8a9b1c4d0dc91b6bab0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 11:43:28 +0000 Subject: [PATCH 139/192] PR comments. --- packages/server/src/api/controllers/auth.ts | 4 ++-- .../server/src/api/controllers/deploy/index.ts | 3 +++ packages/types/src/api/web/deployment.ts | 14 ++++++-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/auth.ts b/packages/server/src/api/controllers/auth.ts index 6aab688c30..0742583a8d 100644 --- a/packages/server/src/api/controllers/auth.ts +++ b/packages/server/src/api/controllers/auth.ts @@ -2,7 +2,7 @@ import { outputProcessing } from "../../utilities/rowProcessor" import { InternalTables } from "../../db/utils" import { getFullUser } from "../../utilities/users" import { roles, context, db as dbCore } from "@budibase/backend-core" -import { AppSelfResponse, ContextUser, Row, UserCtx } from "@budibase/types" +import { AppSelfResponse, ContextUser, UserCtx } from "@budibase/types" import sdk from "../../sdk" import { processUser } from "../../utilities/global" @@ -45,7 +45,7 @@ export async function fetchSelf(ctx: UserCtx) { try { const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA) // specifically needs to make sure is enriched - ctx.body = (await outputProcessing(userTable, user as Row)) as ContextUser + ctx.body = await outputProcessing(userTable, user) } catch (err: any) { let response: ContextUser | {} // user didn't exist in app, don't pretend they do diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 652086f571..b05b82d79a 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -126,6 +126,9 @@ export async function deploymentProgress( try { const db = context.getAppDB() const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS) + if (!deploymentDoc.history?.[ctx.params.deploymentId]) { + ctx.throw(404, "No deployment found") + } ctx.body = deploymentDoc.history?.[ctx.params.deploymentId] } catch (err) { ctx.throw( diff --git a/packages/types/src/api/web/deployment.ts b/packages/types/src/api/web/deployment.ts index 9e4caccc24..f5ed9242b1 100644 --- a/packages/types/src/api/web/deployment.ts +++ b/packages/types/src/api/web/deployment.ts @@ -2,13 +2,11 @@ import { DeploymentDoc, DeploymentStatus } from "../../documents" export interface PublishAppResponse extends DeploymentDoc {} -export type DeploymentProgressResponse = - | { - _id: string - appId: string - status?: DeploymentStatus - updatedAt: number - } - | undefined +export interface DeploymentProgressResponse { + _id: string + appId: string + status?: DeploymentStatus + updatedAt: number +} export type FetchDeploymentResponse = DeploymentProgressResponse[] From e9242dfd1102d6536fbfb598e811435bc83033e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 3 Dec 2024 12:12:38 +0000 Subject: [PATCH 140/192] Fix MySQL options column imports that have commas or other type names in them. --- .../src/api/routes/tests/datasource.spec.ts | 713 ++++++++++-------- packages/server/src/integrations/mysql.ts | 4 +- .../server/src/integrations/utils/utils.ts | 16 +- 3 files changed, 416 insertions(+), 317 deletions(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index f3fac5b99b..ad6a0a2e18 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -169,331 +169,422 @@ const descriptions = datasourceDescribe({ }) if (descriptions.length) { - describe.each(descriptions)("$dbName", ({ config, dsProvider }) => { - let datasource: Datasource - let rawDatasource: Datasource - let client: Knex + describe.each(descriptions)( + "$dbName", + ({ config, dsProvider, isOracle, isMSSQL }) => { + let datasource: Datasource + let rawDatasource: Datasource + let client: Knex - beforeEach(async () => { - const ds = await dsProvider() - rawDatasource = ds.rawDatasource! - datasource = ds.datasource! - client = ds.client! + beforeEach(async () => { + const ds = await dsProvider() + rawDatasource = ds.rawDatasource! + datasource = ds.datasource! + client = ds.client! - jest.clearAllMocks() - nock.cleanAll() - }) - - describe("get", () => { - it("should be able to get a datasource", async () => { - const ds = await config.api.datasource.get(datasource._id!) - expect(ds).toEqual({ - config: expect.any(Object), - plus: datasource.plus, - source: datasource.source, - isSQL: true, - type: "datasource_plus", - _id: datasource._id, - _rev: expect.any(String), - createdAt: expect.any(String), - updatedAt: expect.any(String), - }) + jest.clearAllMocks() + nock.cleanAll() }) - it("should not return database password", async () => { - const ds = await config.api.datasource.get(datasource._id!) - expect(ds.config!.password).toBe("--secret-value--") - }) - }) - - describe("list", () => { - it("returns all the datasources", async () => { - const datasources = await config.api.datasource.fetch() - expect(datasources).toContainEqual(expect.objectContaining(datasource)) - }) - }) - - describe("put", () => { - it("should update an existing datasource", async () => { - const newName = generator.guid() - datasource.name = newName - const updatedDs = await config.api.datasource.update(datasource) - expect(updatedDs.name).toEqual(newName) - expect(events.datasource.updated).toHaveBeenCalledTimes(1) - }) - - it("should not overwrite database password with --secret-value--", async () => { - const password = await context.doInAppContext( - config.getAppId(), - async () => { - const ds = await sdk.datasources.get(datasource._id!) - return ds.config!.password - } - ) - - expect(password).not.toBe("--secret-value--") - - const ds = await config.api.datasource.get(datasource._id!) - expect(ds.config!.password).toBe("--secret-value--") - - await config.api.datasource.update( - await config.api.datasource.get(datasource._id!) - ) - - const newPassword = await context.doInAppContext( - config.getAppId(), - async () => { - const ds = await sdk.datasources.get(datasource._id!) - return ds.config!.password - } - ) - - expect(newPassword).not.toBe("--secret-value--") - expect(newPassword).toBe(password) - }) - }) - - describe("destroy", () => { - it("deletes queries for the datasource after deletion and returns a success message", async () => { - await config.api.query.save({ - datasourceId: datasource._id!, - name: "Test Query", - parameters: [], - fields: {}, - schema: {}, - queryVerb: "read", - transformer: null, - readable: true, - }) - - await config.api.datasource.delete(datasource) - const datasources = await config.api.datasource.fetch() - expect(datasources).not.toContainEqual( - expect.objectContaining(datasource) - ) - expect(events.datasource.deleted).toHaveBeenCalledTimes(1) - }) - }) - - describe("schema", () => { - it("fetching schema will not drop tables or columns", async () => { - const datasourceId = datasource!._id! - - const simpleTable = await config.api.table.save( - tableForDatasource(datasource, { - name: "simple", - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, + describe("get", () => { + it("should be able to get a datasource", async () => { + const ds = await config.api.datasource.get(datasource._id!) + expect(ds).toEqual({ + config: expect.any(Object), + plus: datasource.plus, + source: datasource.source, + isSQL: true, + type: "datasource_plus", + _id: datasource._id, + _rev: expect.any(String), + createdAt: expect.any(String), + updatedAt: expect.any(String), }) - ) - - const stringName = "string" - const fullSchema: { - [type in SupportedSqlTypes]: FieldSchema & { type: type } - } = { - [FieldType.STRING]: { - name: stringName, - type: FieldType.STRING, - }, - [FieldType.LONGFORM]: { - name: "longform", - type: FieldType.LONGFORM, - }, - [FieldType.OPTIONS]: { - name: "options", - type: FieldType.OPTIONS, - constraints: { - presence: { - allowEmpty: false, - }, - inclusion: [], - }, - }, - [FieldType.NUMBER]: { - name: "number", - type: FieldType.NUMBER, - }, - [FieldType.BOOLEAN]: { - name: "boolean", - type: FieldType.BOOLEAN, - }, - [FieldType.ARRAY]: { - name: "array", - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - inclusion: [], - }, - }, - [FieldType.DATETIME]: { - name: "datetime", - type: FieldType.DATETIME, - dateOnly: true, - timeOnly: false, - }, - [FieldType.LINK]: { - name: "link", - type: FieldType.LINK, - tableId: simpleTable._id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "link", - }, - [FieldType.FORMULA]: { - name: "formula", - type: FieldType.FORMULA, - formula: "any formula", - }, - [FieldType.BARCODEQR]: { - name: "barcodeqr", - type: FieldType.BARCODEQR, - }, - [FieldType.BIGINT]: { - name: "bigint", - type: FieldType.BIGINT, - }, - [FieldType.BB_REFERENCE]: { - name: "bb_reference", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, - [FieldType.BB_REFERENCE_SINGLE]: { - name: "bb_reference_single", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - } - - await config.api.table.save( - tableForDatasource(datasource, { - name: "full", - schema: fullSchema, - }) - ) - - const persisted = await config.api.datasource.get(datasourceId) - await config.api.datasource.fetchSchema({ datasourceId }) - - const updated = await config.api.datasource.get(datasourceId) - const expected: Datasource = { - ...persisted, - entities: - persisted?.entities && - Object.entries(persisted.entities).reduce>( - (acc, [tableName, table]) => { - acc[tableName] = expect.objectContaining({ - ...table, - primaryDisplay: expect.not.stringMatching( - new RegExp(`^${table.primaryDisplay || ""}$`) - ), - schema: Object.entries(table.schema).reduce( - (acc, [fieldName, field]) => { - acc[fieldName] = { - ...field, - externalType: allowUndefined(expect.any(String)), - constraints: allowUndefined(expect.any(Object)), - autocolumn: allowUndefined(expect.any(Boolean)), - } - return acc - }, - {} - ), - }) - return acc - }, - {} - ), - - _rev: expect.any(String), - updatedAt: expect.any(String), - } - expect(updated).toEqual(expected) - }) - }) - - describe("verify", () => { - it("should be able to verify the connection", async () => { - await config.api.datasource.verify( - { - datasource: rawDatasource, - }, - { - body: { - connected: true, - }, - } - ) - }) - - it("should state an invalid datasource cannot connect", async () => { - await config.api.datasource.verify( - { - datasource: { - ...rawDatasource, - config: { - ...rawDatasource.config, - password: "wrongpassword", - }, - }, - }, - { - body: { - connected: false, - error: /.*/, // error message differs between databases - }, - } - ) - }) - }) - - describe("info", () => { - it("should fetch information about a datasource with a single table", async () => { - const existingTableNames = ( - await config.api.datasource.info(datasource) - ).tableNames - - const tableName = generator.guid() - await client.schema.createTable(tableName, table => { - table.increments("id").primary() - table.string("name") }) - const info = await config.api.datasource.info(datasource) - expect(info.tableNames).toEqual( - expect.arrayContaining([tableName, ...existingTableNames]) - ) - expect(info.tableNames).toHaveLength(existingTableNames.length + 1) + it("should not return database password", async () => { + const ds = await config.api.datasource.get(datasource._id!) + expect(ds.config!.password).toBe("--secret-value--") + }) }) - it("should fetch information about a datasource with multiple tables", async () => { - const existingTableNames = ( - await config.api.datasource.info(datasource) - ).tableNames + describe("list", () => { + it("returns all the datasources", async () => { + const datasources = await config.api.datasource.fetch() + expect(datasources).toContainEqual( + expect.objectContaining(datasource) + ) + }) + }) - const tableNames = [ - generator.guid(), - generator.guid(), - generator.guid(), - generator.guid(), - ] - for (const tableName of tableNames) { + describe("put", () => { + it("should update an existing datasource", async () => { + const newName = generator.guid() + datasource.name = newName + const updatedDs = await config.api.datasource.update(datasource) + expect(updatedDs.name).toEqual(newName) + expect(events.datasource.updated).toHaveBeenCalledTimes(1) + }) + + it("should not overwrite database password with --secret-value--", async () => { + const password = await context.doInAppContext( + config.getAppId(), + async () => { + const ds = await sdk.datasources.get(datasource._id!) + return ds.config!.password + } + ) + + expect(password).not.toBe("--secret-value--") + + const ds = await config.api.datasource.get(datasource._id!) + expect(ds.config!.password).toBe("--secret-value--") + + await config.api.datasource.update( + await config.api.datasource.get(datasource._id!) + ) + + const newPassword = await context.doInAppContext( + config.getAppId(), + async () => { + const ds = await sdk.datasources.get(datasource._id!) + return ds.config!.password + } + ) + + expect(newPassword).not.toBe("--secret-value--") + expect(newPassword).toBe(password) + }) + }) + + describe("destroy", () => { + it("deletes queries for the datasource after deletion and returns a success message", async () => { + await config.api.query.save({ + datasourceId: datasource._id!, + name: "Test Query", + parameters: [], + fields: {}, + schema: {}, + queryVerb: "read", + transformer: null, + readable: true, + }) + + await config.api.datasource.delete(datasource) + const datasources = await config.api.datasource.fetch() + expect(datasources).not.toContainEqual( + expect.objectContaining(datasource) + ) + expect(events.datasource.deleted).toHaveBeenCalledTimes(1) + }) + }) + + describe("schema", () => { + it("fetching schema will not drop tables or columns", async () => { + const datasourceId = datasource!._id! + + const simpleTable = await config.api.table.save( + tableForDatasource(datasource, { + name: "simple", + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const stringName = "string" + const fullSchema: { + [type in SupportedSqlTypes]: FieldSchema & { type: type } + } = { + [FieldType.STRING]: { + name: stringName, + type: FieldType.STRING, + }, + [FieldType.LONGFORM]: { + name: "longform", + type: FieldType.LONGFORM, + }, + [FieldType.OPTIONS]: { + name: "options", + type: FieldType.OPTIONS, + constraints: { + presence: { + allowEmpty: false, + }, + inclusion: ["1", "2", "3"], + }, + }, + [FieldType.NUMBER]: { + name: "number", + type: FieldType.NUMBER, + }, + [FieldType.BOOLEAN]: { + name: "boolean", + type: FieldType.BOOLEAN, + }, + [FieldType.ARRAY]: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: [], + }, + }, + [FieldType.DATETIME]: { + name: "datetime", + type: FieldType.DATETIME, + dateOnly: true, + timeOnly: false, + }, + [FieldType.LINK]: { + name: "link", + type: FieldType.LINK, + tableId: simpleTable._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "link", + }, + [FieldType.FORMULA]: { + name: "formula", + type: FieldType.FORMULA, + formula: "any formula", + }, + [FieldType.BARCODEQR]: { + name: "barcodeqr", + type: FieldType.BARCODEQR, + }, + [FieldType.BIGINT]: { + name: "bigint", + type: FieldType.BIGINT, + }, + [FieldType.BB_REFERENCE]: { + name: "bb_reference", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + [FieldType.BB_REFERENCE_SINGLE]: { + name: "bb_reference_single", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + } + + await config.api.table.save( + tableForDatasource(datasource, { + name: "full", + schema: fullSchema, + }) + ) + + const persisted = await config.api.datasource.get(datasourceId) + await config.api.datasource.fetchSchema({ datasourceId }) + + const updated = await config.api.datasource.get(datasourceId) + const expected: Datasource = { + ...persisted, + entities: + persisted?.entities && + Object.entries(persisted.entities).reduce>( + (acc, [tableName, table]) => { + acc[tableName] = expect.objectContaining({ + ...table, + primaryDisplay: expect.not.stringMatching( + new RegExp(`^${table.primaryDisplay || ""}$`) + ), + schema: Object.entries(table.schema).reduce( + (acc, [fieldName, field]) => { + acc[fieldName] = { + ...field, + externalType: allowUndefined(expect.any(String)), + constraints: allowUndefined(expect.any(Object)), + autocolumn: allowUndefined(expect.any(Boolean)), + } + return acc + }, + {} + ), + }) + return acc + }, + {} + ), + + _rev: expect.any(String), + updatedAt: expect.any(String), + } + expect(updated).toEqual(expected) + }) + + !isOracle && + !isMSSQL && + it("can fetch options columns with a large number of options", async () => { + const enumOptions = new Array(1000) + .fill(0) + .map((_, i) => i.toString()) + .toSorted() + await client.schema.createTable("options", table => { + table.increments("id").primary() + table.enum("enum", enumOptions, { + useNative: true, + enumName: "enum", + }) + }) + + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource._id!, + }) + expect(resp.errors).toEqual({}) + + const table = resp.datasource.entities!.options + expect( + table.schema.enum.constraints!.inclusion!.toSorted() + ).toEqual(enumOptions) + }) + + !isOracle && + !isMSSQL && + it("can fetch options with commas in them", async () => { + const enumOptions = [ + "Lincoln, Abraham", + "Washington, George", + "Fred", + "Bob", + ].toSorted() + await client.schema.createTable("options", table => { + table.increments("id").primary() + table.enum("enum", enumOptions, { + useNative: true, + enumName: "enum", + }) + }) + + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource._id!, + }) + expect(resp.errors).toEqual({}) + + const table = resp.datasource.entities!.options + expect( + table.schema.enum.constraints!.inclusion!.toSorted() + ).toEqual(enumOptions) + }) + + !isOracle && + !isMSSQL && + it("can fetch options that may include other type names", async () => { + const enumOptions = [ + "int", + "bigint", + "float", + "numeric", + "json", + "map", + ].toSorted() + + await client.schema.createTable("options", table => { + table.increments("id").primary() + table.enum("enum", enumOptions, { + useNative: true, + enumName: "enum", + }) + }) + + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource._id!, + }) + + expect(resp.errors).toEqual({}) + + const table = resp.datasource.entities!.options + expect( + table.schema.enum.constraints!.inclusion!.toSorted() + ).toEqual(enumOptions) + }) + }) + + describe("verify", () => { + it("should be able to verify the connection", async () => { + await config.api.datasource.verify( + { + datasource: rawDatasource, + }, + { + body: { + connected: true, + }, + } + ) + }) + + it("should state an invalid datasource cannot connect", async () => { + await config.api.datasource.verify( + { + datasource: { + ...rawDatasource, + config: { + ...rawDatasource.config, + password: "wrongpassword", + }, + }, + }, + { + body: { + connected: false, + error: /.*/, // error message differs between databases + }, + } + ) + }) + }) + + describe("info", () => { + it("should fetch information about a datasource with a single table", async () => { + const existingTableNames = ( + await config.api.datasource.info(datasource) + ).tableNames + + const tableName = generator.guid() await client.schema.createTable(tableName, table => { table.increments("id").primary() table.string("name") }) - } - const info = await config.api.datasource.info(datasource) - expect(info.tableNames).toEqual( - expect.arrayContaining([...tableNames, ...existingTableNames]) - ) - expect(info.tableNames).toHaveLength( - existingTableNames.length + tableNames.length - ) + const info = await config.api.datasource.info(datasource) + expect(info.tableNames).toEqual( + expect.arrayContaining([tableName, ...existingTableNames]) + ) + expect(info.tableNames).toHaveLength(existingTableNames.length + 1) + }) + + it("should fetch information about a datasource with multiple tables", async () => { + const existingTableNames = ( + await config.api.datasource.info(datasource) + ).tableNames + + const tableNames = [ + generator.guid(), + generator.guid(), + generator.guid(), + generator.guid(), + ] + for (const tableName of tableNames) { + await client.schema.createTable(tableName, table => { + table.increments("id").primary() + table.string("name") + }) + } + + const info = await config.api.datasource.info(datasource) + expect(info.tableNames).toEqual( + expect.arrayContaining([...tableNames, ...existingTableNames]) + ) + expect(info.tableNames).toHaveLength( + existingTableNames.length + tableNames.length + ) + }) }) - }) - }) + } + ) } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 9236409375..34da3dd3c8 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -322,9 +322,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { presence: required && !isAuto && !hasDefault, externalType: column.Type, options: column.Type.startsWith("enum") - ? column.Type.substring(5, column.Type.length - 1) - .split(",") - .map(str => str.replace(/^'(.*)'$/, "$1")) + ? column.Type.substring(6, column.Type.length - 2).split("','") : undefined, }) } diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 73b2f637f8..3c0d82db7e 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -139,11 +139,21 @@ export function generateColumnDefinition(config: { let foundType = FieldType.STRING const lowerCaseType = externalType.toLowerCase() let matchingTypes = [] - for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { - if (lowerCaseType.includes(external)) { - matchingTypes.push({ external, internal }) + + // In at least MySQL, the external type of an ENUM column is "enum('option1', + // 'option2', ...)", which can potentially contain any type name as a + // substring. To get around this interfering with the loop below, we first + // check for an enum column and handle that separately. + if (lowerCaseType.startsWith("enum")) { + matchingTypes.push({ external: "enum", internal: FieldType.OPTIONS }) + } else { + for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { + if (lowerCaseType.includes(external)) { + matchingTypes.push({ external, internal }) + } } } + // Set the foundType based the longest match if (matchingTypes.length > 0) { foundType = matchingTypes.reduce((acc, val) => { From d058b63ea823b9800a93f1a0e8f3d44cfa5726af Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 3 Dec 2024 12:23:07 +0000 Subject: [PATCH 141/192] Fix types. --- packages/server/src/integrations/utils/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index 3c0d82db7e..315a8010e8 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -138,7 +138,7 @@ export function generateColumnDefinition(config: { let { externalType, autocolumn, name, presence, options } = config let foundType = FieldType.STRING const lowerCaseType = externalType.toLowerCase() - let matchingTypes = [] + let matchingTypes: { external: string; internal: PrimitiveTypes }[] = [] // In at least MySQL, the external type of an ENUM column is "enum('option1', // 'option2', ...)", which can potentially contain any type name as a From 582d05651d0cc7d7240d19869ff5f060f59bb872 Mon Sep 17 00:00:00 2001 From: andz-bb Date: Tue, 3 Dec 2024 15:50:51 +0000 Subject: [PATCH 142/192] sort table names alphabetically when selecting data source --- .../DataSourceSelect/DataSourceSelect.svelte | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte index 410af53648..9eef8c1c06 100644 --- a/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte @@ -52,9 +52,16 @@ let modal $: text = value?.label ?? "Choose an option" - $: tables = $tablesStore.list.map(table => - format.table(table, $datasources.list) - ) + $: tables = $tablesStore.list + .map(table => format.table(table, $datasources.list)) + .sort((a, b) => { + // sort tables alphabetically, grouped by datasource + const dsComparison = a.datasourceName.localeCompare(b.datasourceName) + if (dsComparison !== 0) { + return dsComparison + } + return a.label.localeCompare(b.label) + }) $: viewsV1 = $viewsStore.list.map(view => ({ ...view, label: view.name, From 7e22634b52d1432ad91ebca6157013809a52786c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 3 Dec 2024 15:56:34 +0000 Subject: [PATCH 143/192] Bump version to 3.2.20 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 814102c86a..da3a7d0e41 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.19", + "version": "3.2.20", "npmClient": "yarn", "concurrency": 20, "command": { From 75f61b5a31c191aca5b095da39eb6810e5e2e7a9 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 3 Dec 2024 16:11:13 +0000 Subject: [PATCH 144/192] Bump version to 3.2.21 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index da3a7d0e41..1b662e1045 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.20", + "version": "3.2.21", "npmClient": "yarn", "concurrency": 20, "command": { From b1dc9973902551218a0427ad2c3721a4dfe017b6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 16:28:04 +0000 Subject: [PATCH 145/192] old migration typing. --- .../server/src/api/controllers/migrations.ts | 17 +++++++++++++---- packages/types/src/api/web/global/index.ts | 1 + .../types/src/api/web/global/oldMigration.ts | 9 +++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 packages/types/src/api/web/global/oldMigration.ts diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 89255e89b1..a3ff0c5810 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,24 +1,33 @@ import { context } from "@budibase/backend-core" import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { Ctx } from "@budibase/types" +import { + Ctx, + FetchOldMigrationResponse, + GetOldMigrationStatus, + RunOldMigrationRequest, +} from "@budibase/types" import { getAppMigrationVersion, getLatestEnabledMigrationId, } from "../../appMigrations" -export async function migrate(ctx: Ctx) { +export async function migrate(ctx: Ctx) { const options = ctx.request.body // don't await as can take a while, just return migrationImpl(options) ctx.status = 200 } -export async function fetchDefinitions(ctx: Ctx) { +export async function fetchDefinitions( + ctx: Ctx +) { ctx.body = MIGRATIONS ctx.status = 200 } -export async function getMigrationStatus(ctx: Ctx) { +export async function getMigrationStatus( + ctx: Ctx +) { const appId = context.getAppId() if (!appId) { diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts index efcb6dc39c..c43f2928d8 100644 --- a/packages/types/src/api/web/global/index.ts +++ b/packages/types/src/api/web/global/index.ts @@ -4,3 +4,4 @@ export * from "./events" export * from "./configs" export * from "./scim" export * from "./license" +export * from "./oldMigration" diff --git a/packages/types/src/api/web/global/oldMigration.ts b/packages/types/src/api/web/global/oldMigration.ts new file mode 100644 index 0000000000..4b8ce9cc8d --- /dev/null +++ b/packages/types/src/api/web/global/oldMigration.ts @@ -0,0 +1,9 @@ +import { Migration, MigrationOptions } from "../../../sdk" + +export interface RunOldMigrationRequest extends MigrationOptions {} + +export type FetchOldMigrationResponse = Migration[] + +export interface GetOldMigrationStatus { + migrated: boolean +} From 4bf23b0163ace4241d3d9d0ef089f7418d6689cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 16:36:48 +0000 Subject: [PATCH 146/192] Ops API typing --- packages/server/src/api/controllers/ops.ts | 17 ++++------------- packages/types/src/api/web/system/index.ts | 1 + packages/types/src/api/web/system/ops.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 packages/types/src/api/web/system/ops.ts diff --git a/packages/server/src/api/controllers/ops.ts b/packages/server/src/api/controllers/ops.ts index 89ad0ac97b..dc331bb3a3 100644 --- a/packages/server/src/api/controllers/ops.ts +++ b/packages/server/src/api/controllers/ops.ts @@ -1,16 +1,7 @@ -import { Ctx } from "@budibase/types" +import { Ctx, LogOpsRequest, ErrorOpsRequest } from "@budibase/types" import { logging } from "@budibase/backend-core" -interface LogRequest { - message: string - data?: any -} - -interface ErrorRequest { - message: string -} - -export async function log(ctx: Ctx) { +export async function log(ctx: Ctx) { const body = ctx.request.body console.trace(body.message, body.data) console.debug(body.message, body.data) @@ -20,13 +11,13 @@ export async function log(ctx: Ctx) { ctx.status = 204 } -export async function alert(ctx: Ctx) { +export async function alert(ctx: Ctx) { const body = ctx.request.body logging.logAlert(body.message, new Error(body.message)) ctx.status = 204 } -export async function error(ctx: Ctx) { +export async function error(ctx: Ctx) { const body = ctx.request.body throw new Error(body.message) } diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index 8820bf5cd1..0ce2738ddb 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -1,2 +1,3 @@ export * from "./environment" export * from "./status" +export * from "./ops" diff --git a/packages/types/src/api/web/system/ops.ts b/packages/types/src/api/web/system/ops.ts new file mode 100644 index 0000000000..390e4db006 --- /dev/null +++ b/packages/types/src/api/web/system/ops.ts @@ -0,0 +1,8 @@ +export interface LogOpsRequest { + message: string + data?: any +} + +export interface ErrorOpsRequest { + message: string +} From 78ea6e73fd6bfdb3c95a7e60f70ecdc7ec645429 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Dec 2024 16:42:36 +0000 Subject: [PATCH 147/192] adding buffer settings for single image --- hosting/single/nginx/nginx-default-site.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hosting/single/nginx/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf index d2f8e229c6..f72d33e236 100644 --- a/hosting/single/nginx/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -46,6 +46,11 @@ server { } location ~ ^/api/(system|admin|global)/ { + # Enable buffering for potentially large OIDC configs + proxy_buffering on; + proxy_buffer_size 16k; + proxy_buffers 4 32k; + proxy_pass http://127.0.0.1:4002; } From b1868d7701dbdcaaaae287b6de0c4b66bff14c11 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:43:13 +0000 Subject: [PATCH 148/192] Remove unused types (#15108) --- packages/pro | 2 +- packages/types/src/api/account/index.ts | 1 - packages/types/src/api/account/user.ts | 8 -------- 3 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 packages/types/src/api/account/user.ts diff --git a/packages/pro b/packages/pro index d9245f3d6d..e60f4b1b36 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d9245f3d6d0b41ec2e6b3406b791f9e7448882cb +Subproject commit e60f4b1b364fd49d2bb082f298757f83cb2032f0 diff --git a/packages/types/src/api/account/index.ts b/packages/types/src/api/account/index.ts index 1f46e364d8..e5bdd6da54 100644 --- a/packages/types/src/api/account/index.ts +++ b/packages/types/src/api/account/index.ts @@ -1,4 +1,3 @@ export * from "./accounts" -export * from "./user" export * from "./license" export * from "./status" diff --git a/packages/types/src/api/account/user.ts b/packages/types/src/api/account/user.ts deleted file mode 100644 index 0a0ab0aeb8..0000000000 --- a/packages/types/src/api/account/user.ts +++ /dev/null @@ -1,8 +0,0 @@ -export interface PostAccountUserActivity { - timestamp: number -} - -export interface PostAccountUserActivityResponse { - userId: string - timestamp: number -} From 5f28689800cc352ffe7d3da43f1ad125f6d1d930 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 3 Dec 2024 16:45:39 +0000 Subject: [PATCH 149/192] Bump version to 3.2.22 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1b662e1045..e67812f37b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.21", + "version": "3.2.22", "npmClient": "yarn", "concurrency": 20, "command": { From d0179ed815af1554acfbcd98c9fac5819a2721cc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 16:50:13 +0000 Subject: [PATCH 150/192] Permission API typing. --- .../backend-core/src/security/permissions.ts | 58 +++++++++---------- .../src/security/tests/permissions.spec.ts | 2 +- .../server/src/api/controllers/permission.ts | 8 ++- packages/types/src/api/web/app/permission.ts | 6 +- packages/types/src/sdk/permissions.ts | 19 ++++++ 5 files changed, 58 insertions(+), 35 deletions(-) diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 929ae92909..263f042a9d 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -2,6 +2,8 @@ import { PermissionLevel, PermissionType, BuiltinPermissionID, + Permission, + BuiltinPermissions, } from "@budibase/types" import flatten from "lodash/flatten" import cloneDeep from "lodash/fp/cloneDeep" @@ -12,7 +14,7 @@ export type RoleHierarchy = { permissionId: string }[] -export class Permission { +export class PermissionImpl implements Permission { type: PermissionType level: PermissionLevel @@ -61,68 +63,62 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] { } } -export const BUILTIN_PERMISSIONS: { - [key in keyof typeof BuiltinPermissionID]: { - _id: (typeof BuiltinPermissionID)[key] - name: string - permissions: Permission[] - } -} = { +export const BUILTIN_PERMISSIONS: BuiltinPermissions = { PUBLIC: { _id: BuiltinPermissionID.PUBLIC, name: "Public", permissions: [ - new Permission(PermissionType.WEBHOOK, PermissionLevel.EXECUTE), + new PermissionImpl(PermissionType.WEBHOOK, PermissionLevel.EXECUTE), ], }, READ_ONLY: { _id: BuiltinPermissionID.READ_ONLY, name: "Read only", permissions: [ - new Permission(PermissionType.QUERY, PermissionLevel.READ), - new Permission(PermissionType.TABLE, PermissionLevel.READ), - new Permission(PermissionType.APP, PermissionLevel.READ), + new PermissionImpl(PermissionType.QUERY, PermissionLevel.READ), + new PermissionImpl(PermissionType.TABLE, PermissionLevel.READ), + new PermissionImpl(PermissionType.APP, PermissionLevel.READ), ], }, WRITE: { _id: BuiltinPermissionID.WRITE, name: "Read/Write", permissions: [ - new Permission(PermissionType.QUERY, PermissionLevel.WRITE), - new Permission(PermissionType.TABLE, PermissionLevel.WRITE), - new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), - new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ), - new Permission(PermissionType.APP, PermissionLevel.READ), + new PermissionImpl(PermissionType.QUERY, PermissionLevel.WRITE), + new PermissionImpl(PermissionType.TABLE, PermissionLevel.WRITE), + new PermissionImpl(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), + new PermissionImpl(PermissionType.LEGACY_VIEW, PermissionLevel.READ), + new PermissionImpl(PermissionType.APP, PermissionLevel.READ), ], }, POWER: { _id: BuiltinPermissionID.POWER, name: "Power", permissions: [ - new Permission(PermissionType.TABLE, PermissionLevel.WRITE), - new Permission(PermissionType.USER, PermissionLevel.READ), - new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), - new Permission(PermissionType.WEBHOOK, PermissionLevel.READ), - new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ), - new Permission(PermissionType.APP, PermissionLevel.READ), + new PermissionImpl(PermissionType.TABLE, PermissionLevel.WRITE), + new PermissionImpl(PermissionType.USER, PermissionLevel.READ), + new PermissionImpl(PermissionType.AUTOMATION, PermissionLevel.EXECUTE), + new PermissionImpl(PermissionType.WEBHOOK, PermissionLevel.READ), + new PermissionImpl(PermissionType.LEGACY_VIEW, PermissionLevel.READ), + new PermissionImpl(PermissionType.APP, PermissionLevel.READ), ], }, ADMIN: { _id: BuiltinPermissionID.ADMIN, name: "Admin", permissions: [ - new Permission(PermissionType.TABLE, PermissionLevel.ADMIN), - new Permission(PermissionType.USER, PermissionLevel.ADMIN), - new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN), - new Permission(PermissionType.WEBHOOK, PermissionLevel.READ), - new Permission(PermissionType.QUERY, PermissionLevel.ADMIN), - new Permission(PermissionType.LEGACY_VIEW, PermissionLevel.READ), - new Permission(PermissionType.APP, PermissionLevel.READ), + new PermissionImpl(PermissionType.TABLE, PermissionLevel.ADMIN), + new PermissionImpl(PermissionType.USER, PermissionLevel.ADMIN), + new PermissionImpl(PermissionType.AUTOMATION, PermissionLevel.ADMIN), + new PermissionImpl(PermissionType.WEBHOOK, PermissionLevel.READ), + new PermissionImpl(PermissionType.QUERY, PermissionLevel.ADMIN), + new PermissionImpl(PermissionType.LEGACY_VIEW, PermissionLevel.READ), + new PermissionImpl(PermissionType.APP, PermissionLevel.READ), ], }, } -export function getBuiltinPermissions() { +export function getBuiltinPermissions(): BuiltinPermissions { return cloneDeep(BUILTIN_PERMISSIONS) } diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts index f98833c7cd..d268a56a1f 100644 --- a/packages/backend-core/src/security/tests/permissions.spec.ts +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -133,7 +133,7 @@ describe("getBuiltinPermissionByID", () => { _id: BuiltinPermissionID.PUBLIC, name: "Public", permissions: [ - new permissions.Permission( + new permissions.PermissionImpl( permissions.PermissionType.WEBHOOK, permissions.PermissionLevel.EXECUTE ), diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index ead48d3db8..2ef197dbca 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -9,6 +9,8 @@ import { RemovePermissionRequest, RemovePermissionResponse, FetchResourcePermissionInfoResponse, + FetchBuiltinPermissionsRequest, + FetchPermissionLevelsRequest, } from "@budibase/types" import { CURRENTLY_SUPPORTED_LEVELS, @@ -19,11 +21,13 @@ import { PermissionUpdateType } from "../../sdk/app/permissions" const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -export function fetchBuiltin(ctx: UserCtx) { +export function fetchBuiltin( + ctx: UserCtx +) { ctx.body = Object.values(permissions.getBuiltinPermissions()) } -export function fetchLevels(ctx: UserCtx) { +export function fetchLevels(ctx: UserCtx) { // for now only provide the read/write perms externally ctx.body = SUPPORTED_LEVELS } diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts index b40310f21c..1a19fb0834 100644 --- a/packages/types/src/api/web/app/permission.ts +++ b/packages/types/src/api/web/app/permission.ts @@ -1,4 +1,8 @@ -import { PermissionLevel } from "../../../sdk" +import { BuiltinPermission, PermissionLevel } from "../../../sdk" + +export type FetchBuiltinPermissionsRequest = BuiltinPermission[] + +export type FetchPermissionLevelsRequest = string[] export interface FetchResourcePermissionInfoResponse { [key: string]: Record diff --git a/packages/types/src/sdk/permissions.ts b/packages/types/src/sdk/permissions.ts index bb36af4170..43336a3317 100644 --- a/packages/types/src/sdk/permissions.ts +++ b/packages/types/src/sdk/permissions.ts @@ -36,3 +36,22 @@ export enum PermissionSource { INHERITED = "INHERITED", BASE = "BASE", } + +export interface Permission { + type: PermissionType + level: PermissionLevel +} + +export interface BuiltinPermission { + _id: BuiltinPermissionID + name: string + permissions: Permission[] +} + +export type BuiltinPermissions = { + [key in keyof typeof BuiltinPermissionID]: { + _id: (typeof BuiltinPermissionID)[key] + name: string + permissions: Permission[] + } +} From 3446b1c67806c4e5fa10d9247b317ce770632d24 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 17:02:30 +0000 Subject: [PATCH 151/192] Plugin API typing. --- .../server/src/api/controllers/plugin/file.ts | 6 +++- .../src/api/controllers/plugin/index.ts | 33 ++++++++++++------- packages/server/src/sdk/plugins/plugins.ts | 2 +- packages/types/src/api/web/plugins.ts | 14 +++++++- packages/types/src/documents/global/plugin.ts | 4 +-- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/packages/server/src/api/controllers/plugin/file.ts b/packages/server/src/api/controllers/plugin/file.ts index 3b21f08387..0f40998617 100644 --- a/packages/server/src/api/controllers/plugin/file.ts +++ b/packages/server/src/api/controllers/plugin/file.ts @@ -3,8 +3,12 @@ import { getPluginMetadata, extractTarball, } from "../../../utilities/fileSystem" +import { FileType } from "@budibase/types" -export async function fileUpload(file: { name: string; path: string }) { +export async function fileUpload(file: FileType) { + if (!file.name || !file.path) { + throw new Error("File is not valid - cannot upload.") + } if (!file.name.endsWith(".tar.gz")) { throw new Error("Plugin must be compressed into a gzipped tarball.") } diff --git a/packages/server/src/api/controllers/plugin/index.ts b/packages/server/src/api/controllers/plugin/index.ts index e1c51f0219..2a4d69b671 100644 --- a/packages/server/src/api/controllers/plugin/index.ts +++ b/packages/server/src/api/controllers/plugin/index.ts @@ -2,26 +2,37 @@ import { npmUpload, urlUpload, githubUpload } from "./uploaders" import { plugins as pluginCore } from "@budibase/backend-core" import { PluginType, - FileType, PluginSource, - Ctx, CreatePluginRequest, CreatePluginResponse, + UserCtx, + UploadPluginRequest, + Plugin, + UploadPluginResponse, + FetchPluginResponse, + DeletePluginResponse, } from "@budibase/types" import env from "../../../environment" import { clientAppSocket } from "../../../websockets" import sdk from "../../../sdk" import { sdk as pro } from "@budibase/pro" -export async function upload(ctx: any) { - const plugins: FileType[] = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] +export async function upload( + ctx: UserCtx +) { + const files = ctx.request.files + const plugins = + files && Array.isArray(files.file) && files.file.length > 1 + ? Array.from(files.file) + : [files?.file] + try { - let docs = [] + let docs: Plugin[] = [] // can do single or multiple plugins for (let plugin of plugins) { + if (!plugin || Array.isArray(plugin)) { + continue + } const doc = await sdk.plugins.processUploaded(plugin, PluginSource.FILE) docs.push(doc) } @@ -37,7 +48,7 @@ export async function upload(ctx: any) { } export async function create( - ctx: Ctx + ctx: UserCtx ) { const { source, url, headers, githubToken } = ctx.request.body @@ -91,11 +102,11 @@ export async function create( } } -export async function fetch(ctx: any) { +export async function fetch(ctx: UserCtx) { ctx.body = await sdk.plugins.fetch() } -export async function destroy(ctx: any) { +export async function destroy(ctx: UserCtx) { const { pluginId } = ctx.params try { diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts index f6c26ca339..650065b40c 100644 --- a/packages/server/src/sdk/plugins/plugins.ts +++ b/packages/server/src/sdk/plugins/plugins.ts @@ -10,7 +10,7 @@ import env from "../../environment" import { clientAppSocket } from "../../websockets" import { sdk as pro } from "@budibase/pro" -export async function fetch(type?: PluginType) { +export async function fetch(type?: PluginType): Promise { const db = tenancy.getGlobalDB() const response = await db.allDocs( dbCore.getPluginParams(null, { diff --git a/packages/types/src/api/web/plugins.ts b/packages/types/src/api/web/plugins.ts index 458ad3f6ce..4dd47550e5 100644 --- a/packages/types/src/api/web/plugins.ts +++ b/packages/types/src/api/web/plugins.ts @@ -1,4 +1,10 @@ -import { PluginSource } from "../../documents" +import { PluginSource, Plugin } from "../../documents" + +export interface UploadPluginRequest {} +export interface UploadPluginResponse { + message: string + plugins: Plugin[] +} export interface CreatePluginRequest { source: PluginSource @@ -10,3 +16,9 @@ export interface CreatePluginRequest { export interface CreatePluginResponse { plugin: any } + +export type FetchPluginResponse = Plugin[] + +export interface DeletePluginResponse { + message: string +} diff --git a/packages/types/src/documents/global/plugin.ts b/packages/types/src/documents/global/plugin.ts index dd96bc20db..5a7c701ae5 100644 --- a/packages/types/src/documents/global/plugin.ts +++ b/packages/types/src/documents/global/plugin.ts @@ -13,8 +13,8 @@ export enum PluginSource { FILE = "File Upload", } export interface FileType { - path: string - name: string + path: string | null + name: string | null } export interface Plugin extends Document { From 23d9808cb61cf045e9347863df1ebaf4bfb7bd71 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 17:29:41 +0000 Subject: [PATCH 152/192] Query API typing. --- .../server/src/api/controllers/query/index.ts | 79 ++++++++++++------- packages/server/src/api/routes/query.ts | 2 +- .../src/automations/steps/executeQuery.ts | 3 +- .../server/src/tests/utilities/api/query.ts | 6 +- packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/app/query.ts | 47 +++++++++++ packages/types/src/api/web/index.ts | 1 - packages/types/src/api/web/query.ts | 20 ----- 8 files changed, 104 insertions(+), 55 deletions(-) create mode 100644 packages/types/src/api/web/app/query.ts delete mode 100644 packages/types/src/api/web/query.ts diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 15c60bcf47..d73bb8c024 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -4,26 +4,38 @@ import { save as saveDatasource } from "../datasource" import { RestImporter } from "./import" import { invalidateCachedVariable } from "../../../threads/utils" import env from "../../../environment" -import { events, context, utils, constants } from "@budibase/backend-core" +import { constants, context, events, utils } from "@budibase/backend-core" import sdk from "../../../sdk" import { QueryEvent, QueryEventParameters } from "../../../threads/definitions" import { ConfigType, - Query, - UserCtx, - SessionCookie, - JsonFieldSubType, - QueryResponse, - QuerySchema, - FieldType, + CreateDatasourceRequest, + Datasource, ExecuteQueryRequest, - ExecuteQueryResponse, + ExecuteV2QueryResponse, + ExecuteV1QueryResponse, + FetchQueriesResponse, + FieldType, + FindQueryResponse, + ImportRestQueryRequest, + ImportRestQueryResponse, + JsonFieldSubType, PreviewQueryRequest, PreviewQueryResponse, + Query, + QueryResponse, + QuerySchema, + SaveQueryRequest, + SaveQueryResponse, + SessionCookie, + SourceName, + UserCtx, + DeleteQueryResponse, } from "@budibase/types" -import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" +import { utils as JsonUtils, ValidQueryNameRegex } from "@budibase/shared-core" import { findHBSBlocks } from "@budibase/string-templates" import { ObjectId } from "mongodb" +import { merge } from "lodash" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: env.QUERY_THREAD_TIMEOUT, @@ -43,11 +55,13 @@ function validateQueryInputs(parameters: QueryEventParameters) { } } -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { ctx.body = await sdk.queries.fetch() } -const _import = async (ctx: UserCtx) => { +const _import = async ( + ctx: UserCtx +) => { const body = ctx.request.body const data = body.data @@ -58,9 +72,9 @@ const _import = async (ctx: UserCtx) => { if (!body.datasourceId) { // construct new datasource const info: any = await importer.getInfo() - let datasource = { + let datasource: Datasource = { type: "datasource", - source: "REST", + source: SourceName.REST, config: { url: info.url, defaultHeaders: [], @@ -69,8 +83,14 @@ const _import = async (ctx: UserCtx) => { name: info.name, } // save the datasource - const datasourceCtx = { ...ctx } - datasourceCtx.request.body.datasource = datasource + const datasourceCtx: UserCtx = merge(ctx, { + request: { + body: { + datasource, + tablesFilter: [], + }, + }, + }) await saveDatasource(datasourceCtx) datasourceId = datasourceCtx.body.datasource._id } else { @@ -88,7 +108,7 @@ const _import = async (ctx: UserCtx) => { } export { _import as import } -export async function save(ctx: UserCtx) { +export async function save(ctx: UserCtx) { const db = context.getAppDB() const query: Query = ctx.request.body @@ -119,10 +139,9 @@ export async function save(ctx: UserCtx) { query._rev = response.rev ctx.body = query - ctx.message = `Query ${query.name} saved successfully.` } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { const queryId = ctx.params.queryId ctx.body = await sdk.queries.find(queryId) } @@ -335,7 +354,7 @@ export async function preview( async function execute( ctx: UserCtx< ExecuteQueryRequest, - ExecuteQueryResponse | Record[] + ExecuteV2QueryResponse | ExecuteV1QueryResponse >, opts: any = { rowsOnly: false, isAutomation: false } ) { @@ -390,19 +409,21 @@ async function execute( } export async function executeV1( - ctx: UserCtx[]> + ctx: UserCtx ) { return execute(ctx, { rowsOnly: true, isAutomation: false }) } export async function executeV2( - ctx: UserCtx< - ExecuteQueryRequest, - ExecuteQueryResponse | Record[] - >, - { isAutomation }: { isAutomation?: boolean } = {} + ctx: UserCtx ) { - return execute(ctx, { rowsOnly: false, isAutomation }) + return execute(ctx, { rowsOnly: false }) +} + +export async function executeV2AsAutomation( + ctx: UserCtx +) { + return execute(ctx, { rowsOnly: false, isAutomation: true }) } const removeDynamicVariables = async (queryId: string) => { @@ -426,14 +447,14 @@ const removeDynamicVariables = async (queryId: string) => { } } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const queryId = ctx.params.queryId as string await removeDynamicVariables(queryId) const query = await db.get(queryId) const datasource = await sdk.datasources.get(query.datasourceId) await db.remove(ctx.params.queryId, ctx.params.revId) - ctx.message = `Query deleted.` + ctx.body = { message: `Query deleted.` } ctx.status = 200 await events.query.deleted(datasource, query) } diff --git a/packages/server/src/api/routes/query.ts b/packages/server/src/api/routes/query.ts index eb857d0637..89d3651227 100644 --- a/packages/server/src/api/routes/query.ts +++ b/packages/server/src/api/routes/query.ts @@ -56,7 +56,7 @@ router "/api/v2/queries/:queryId", paramResource("queryId"), authorized(PermissionType.QUERY, PermissionLevel.WRITE), - queryController.executeV2 as any + queryController.executeV2 ) export default router diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index a42182df37..095c91a44d 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -12,6 +12,7 @@ import { ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" +import { executeV2AsAutomation } from "../../api/controllers/query" export const definition: AutomationStepDefinition = { name: "External Data Connector", @@ -94,7 +95,7 @@ export async function run({ }) try { - await queryController.executeV2(ctx, { isAutomation: true }) + await queryController.executeV2AsAutomation(ctx) const { data, ...rest } = ctx.body return { diff --git a/packages/server/src/tests/utilities/api/query.ts b/packages/server/src/tests/utilities/api/query.ts index 2d5f7970cd..fc81fc4561 100644 --- a/packages/server/src/tests/utilities/api/query.ts +++ b/packages/server/src/tests/utilities/api/query.ts @@ -1,7 +1,7 @@ import { Query, ExecuteQueryRequest, - ExecuteQueryResponse, + ExecuteV2QueryResponse, PreviewQueryRequest, PreviewQueryResponse, } from "@budibase/types" @@ -17,8 +17,8 @@ export class QueryAPI extends TestAPI { queryId: string, body?: ExecuteQueryRequest, expectations?: Expectations - ): Promise => { - return await this._post( + ): Promise => { + return await this._post( `/api/v2/queries/${queryId}`, { body, diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 40c8ebf9ca..82770543df 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -12,3 +12,4 @@ export * from "./automation" export * from "./component" export * from "./integration" export * from "./metadata" +export * from "./query" diff --git a/packages/types/src/api/web/app/query.ts b/packages/types/src/api/web/app/query.ts new file mode 100644 index 0000000000..302f0d03e5 --- /dev/null +++ b/packages/types/src/api/web/app/query.ts @@ -0,0 +1,47 @@ +import { + Datasource, + Query, + QueryPreview, + QuerySchema, +} from "../../../documents" + +export type FetchQueriesResponse = Query[] + +export interface SaveQueryRequest extends Query {} +export interface SaveQueryResponse extends Query {} + +export interface ImportRestQueryRequest { + datasourceId: string + data: string + datasource: Datasource +} +export interface ImportRestQueryResponse { + errorQueries: Query[] + queries: Query[] + datasourceId: string +} + +export interface FindQueryResponse extends Query {} + +export interface PreviewQueryRequest extends QueryPreview {} + +export interface PreviewQueryResponse { + rows: any[] + nestedSchemaFields: { [key: string]: { [key: string]: string | QuerySchema } } + schema: { [key: string]: string | QuerySchema } + info: any + extra: any +} + +export interface ExecuteQueryRequest { + parameters?: Record + pagination?: any +} +export type ExecuteV1QueryResponse = Record[] +export interface ExecuteV2QueryResponse { + data: Record[] +} + +export interface DeleteQueryResponse { + message: string +} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 4021eafee8..dd2ee3beae 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -13,7 +13,6 @@ export * from "./searchFilter" export * from "./cookies" export * from "./automation" export * from "./layout" -export * from "./query" export * from "./role" export * from "./plugins" export * from "./apikeys" diff --git a/packages/types/src/api/web/query.ts b/packages/types/src/api/web/query.ts deleted file mode 100644 index 66f0248647..0000000000 --- a/packages/types/src/api/web/query.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { QueryPreview, QuerySchema } from "../../documents" - -export interface PreviewQueryRequest extends QueryPreview {} - -export interface PreviewQueryResponse { - rows: any[] - nestedSchemaFields: { [key: string]: { [key: string]: string | QuerySchema } } - schema: { [key: string]: string | QuerySchema } - info: any - extra: any -} - -export interface ExecuteQueryRequest { - parameters?: Record - pagination?: any -} - -export interface ExecuteQueryResponse { - data: Record[] -} From efcdd360e3d02b3992ae796fa8d16ecab73d4426 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 17:42:31 +0000 Subject: [PATCH 153/192] Routing API typing. --- packages/server/src/api/controllers/role.ts | 4 ++-- packages/server/src/api/controllers/routing.ts | 18 +++++++++++++----- packages/server/src/utilities/routing/index.ts | 13 ++++--------- packages/types/src/api/web/app/index.ts | 1 + packages/types/src/api/web/app/screen.ts | 8 ++++++++ packages/types/src/api/web/role.ts | 2 +- packages/types/src/documents/app/screen.ts | 17 +++++++++++++++++ 7 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 packages/types/src/api/web/app/screen.ts diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 7f5ac6b92f..30ca70b505 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -9,7 +9,7 @@ import { getUserMetadataParams, InternalTables } from "../../db/utils" import { AccessibleRolesResponse, Database, - DestroyRoleResponse, + DeleteRoleResponse, FetchRolesResponse, FindRoleResponse, Role, @@ -199,7 +199,7 @@ export async function save(ctx: UserCtx) { builderSocket?.emitRoleUpdate(ctx, role) } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() let roleId = ctx.params.roleId as string if (roles.isBuiltin(roleId)) { diff --git a/packages/server/src/api/controllers/routing.ts b/packages/server/src/api/controllers/routing.ts index 040cda4dd0..fc9e58def5 100644 --- a/packages/server/src/api/controllers/routing.ts +++ b/packages/server/src/api/controllers/routing.ts @@ -1,11 +1,17 @@ import { getRoutingInfo } from "../../utilities/routing" import { roles } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" +import { + FetchClientScreenRoutingResponse, + FetchScreenRoutingResponse, + ScreenRoutingJson, + UserCtx, +} from "@budibase/types" const URL_SEPARATOR = "/" class Routing { - json: any + json: ScreenRoutingJson + constructor() { this.json = {} } @@ -43,7 +49,7 @@ class Routing { * @returns The routing structure, this is the full structure designed for use in the builder, * if the client routing is required then the updateRoutingStructureForUserRole should be used. */ -async function getRoutingStructure() { +async function getRoutingStructure(): Promise<{ routes: ScreenRoutingJson }> { const screenRoutes = await getRoutingInfo() const routing = new Routing() @@ -56,11 +62,13 @@ async function getRoutingStructure() { return { routes: routing.json } } -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: UserCtx) { ctx.body = await getRoutingStructure() } -export async function clientFetch(ctx: UserCtx) { +export async function clientFetch( + ctx: UserCtx +) { const routing = await getRoutingStructure() let roleId = ctx.user?.role?._id const roleIds = roleId ? await roles.getUserRoleIdHierarchy(roleId) : [] diff --git a/packages/server/src/utilities/routing/index.ts b/packages/server/src/utilities/routing/index.ts index 82d45743ce..f0086daa72 100644 --- a/packages/server/src/utilities/routing/index.ts +++ b/packages/server/src/utilities/routing/index.ts @@ -1,24 +1,19 @@ import { createRoutingView } from "../../db/views/staticViews" import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils" import { context } from "@budibase/backend-core" -import { ScreenRouting, Document } from "@budibase/types" +import { ScreenRoutesViewOutput } from "@budibase/types" -interface ScreenRoutesView extends Document { - id: string - routing: ScreenRouting -} - -export async function getRoutingInfo(): Promise { +export async function getRoutingInfo(): Promise { const db = context.getAppDB() try { - const allRouting = await db.query( + const allRouting = await db.query( getQueryIndex(ViewName.ROUTING), { startkey: "", endkey: UNICODE_MAX, } ) - return allRouting.rows.map(row => row.value as ScreenRoutesView) + return allRouting.rows.map(row => row.value) } catch (err: any) { // check if the view doesn't exist, it should for all new instances /* istanbul ignore next */ diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 82770543df..15c9c8929d 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -13,3 +13,4 @@ export * from "./component" export * from "./integration" export * from "./metadata" export * from "./query" +export * from "./screen" diff --git a/packages/types/src/api/web/app/screen.ts b/packages/types/src/api/web/app/screen.ts new file mode 100644 index 0000000000..fc9eb63975 --- /dev/null +++ b/packages/types/src/api/web/app/screen.ts @@ -0,0 +1,8 @@ +import { ScreenRoutingJson } from "../../../documents" + +export interface FetchScreenRoutingResponse { + routes: ScreenRoutingJson +} + +export interface FetchClientScreenRoutingResponse + extends FetchScreenRoutingResponse {} diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts index f1c573c849..eeedaea163 100644 --- a/packages/types/src/api/web/role.ts +++ b/packages/types/src/api/web/role.ts @@ -18,7 +18,7 @@ export interface FindRoleResponse extends Role {} export type FetchRolesResponse = Role[] -export interface DestroyRoleResponse { +export interface DeleteRoleResponse { message: string } diff --git a/packages/types/src/documents/app/screen.ts b/packages/types/src/documents/app/screen.ts index 4977c79b0b..b2cedf31a9 100644 --- a/packages/types/src/documents/app/screen.ts +++ b/packages/types/src/documents/app/screen.ts @@ -24,3 +24,20 @@ export interface Screen extends Document { name?: string pluginAdded?: boolean } + +export interface ScreenRoutesViewOutput extends Document { + id: string + routing: ScreenRouting +} + +export type ScreenRoutingJson = Record< + string, + { + subpaths: Record< + string, + { + screens: Record + } + > + } +> From efb99c6446711a1fad35ce8194902047e4fc1bca Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 17:50:51 +0000 Subject: [PATCH 154/192] Row API typing. --- .../server/src/api/controllers/row/index.ts | 28 +++++++++++++------ .../server/src/tests/utilities/api/row.ts | 6 ++-- packages/types/src/api/web/app/index.ts | 1 - packages/types/src/api/web/app/row.ts | 18 ------------ packages/types/src/api/web/app/rows/index.ts | 28 ++++++++++++++++++- 5 files changed, 50 insertions(+), 31 deletions(-) delete mode 100644 packages/types/src/api/web/app/row.ts diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cc5491e54f..0463c0a565 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -11,11 +11,14 @@ import { DeleteRow, DeleteRowRequest, DeleteRows, + DownloadAttachmentResponse, EventType, ExportRowsRequest, ExportRowsResponse, + FetchEnrichedRowResponse, + FetchRowsResponse, FieldType, - GetRowResponse, + FindRowResponse, isRelationshipField, PatchRowRequest, PatchRowResponse, @@ -23,12 +26,15 @@ import { Row, RowAttachment, RowSearchParams, + SaveRowRequest, + SaveRowResponse, SearchFilters, SearchRowRequest, SearchRowResponse, Table, UserCtx, - ValidateResponse, + ValidateRowRequest, + ValidateRowResponse, } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" @@ -83,7 +89,7 @@ export async function patch( } } -export const save = async (ctx: UserCtx) => { +export const save = async (ctx: UserCtx) => { const { tableId, viewId } = utils.getSourceId(ctx) const sourceId = viewId || tableId @@ -131,12 +137,12 @@ export async function fetchLegacyView(ctx: any) { }) } -export async function fetch(ctx: any) { +export async function fetch(ctx: UserCtx) { const { tableId } = utils.getSourceId(ctx) ctx.body = await sdk.rows.fetch(tableId) } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { const { tableId, viewId } = utils.getSourceId(ctx) const sourceId = viewId || tableId const rowId = ctx.params.rowId @@ -314,7 +320,9 @@ function replaceTableNamesInFilters( }) } -export async function validate(ctx: Ctx) { +export async function validate( + ctx: Ctx +) { const source = await utils.getSource(ctx) const table = await utils.getTableFromSource(source) // external tables are hard to validate currently @@ -328,7 +336,9 @@ export async function validate(ctx: Ctx) { } } -export async function fetchEnrichedRow(ctx: UserCtx) { +export async function fetchEnrichedRow( + ctx: UserCtx +) { const { tableId } = utils.getSourceId(ctx) ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx) } @@ -366,7 +376,9 @@ export const exportRows = async ( ctx.body = apiFileReturn(content) } -export async function downloadAttachment(ctx: UserCtx) { +export async function downloadAttachment( + ctx: UserCtx +) { const { columnName } = ctx.params const { tableId } = utils.getSourceId(ctx) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 52317e142a..f68b41aca9 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -2,7 +2,7 @@ import { PatchRowRequest, SaveRowRequest, Row, - ValidateResponse, + ValidateRowResponse, ExportRowsRequest, BulkImportRequest, BulkImportResponse, @@ -51,8 +51,8 @@ export class RowAPI extends TestAPI { sourceId: string, row: SaveRowRequest, expectations?: Expectations - ): Promise => { - return await this._post( + ): Promise => { + return await this._post( `/api/${sourceId}/rows/validate`, { body: row, diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index 15c9c8929d..d130d19a42 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -1,6 +1,5 @@ export * from "./backup" export * from "./datasource" -export * from "./row" export * from "./view" export * from "./rows" export * from "./table" diff --git a/packages/types/src/api/web/app/row.ts b/packages/types/src/api/web/app/row.ts deleted file mode 100644 index 4ab4090461..0000000000 --- a/packages/types/src/api/web/app/row.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Row } from "../../../documents/app/row" - -export interface GetRowResponse extends Row {} - -export interface DeleteRows { - rows: (Row | string)[] -} - -export interface DeleteRow { - _id: string -} - -export type DeleteRowRequest = DeleteRows | DeleteRow - -export interface ValidateResponse { - valid: boolean - errors: Record -} diff --git a/packages/types/src/api/web/app/rows/index.ts b/packages/types/src/api/web/app/rows/index.ts index 2642a8b04e..7994f9736b 100644 --- a/packages/types/src/api/web/app/rows/index.ts +++ b/packages/types/src/api/web/app/rows/index.ts @@ -1,11 +1,17 @@ import { SearchFilters } from "../../../../sdk" import { Row } from "../../../../documents" -import { SortOrder } from "../../../../api/web/pagination" +import { SortOrder } from "../../pagination" import { ReadStream } from "fs" +import stream from "node:stream" export * from "./search" +export interface FetchEnrichedRowResponse extends Row {} + +export type FetchRowsResponse = Row[] + export interface SaveRowRequest extends Row {} +export interface SaveRowResponse extends Row {} export interface PatchRowRequest extends Row { _id: string @@ -26,3 +32,23 @@ export interface ExportRowsRequest { } export type ExportRowsResponse = ReadStream + +export type DownloadAttachmentResponse = stream.PassThrough | stream.Readable + +export interface FindRowResponse extends Row {} + +export interface DeleteRows { + rows: (Row | string)[] +} + +export interface DeleteRow { + _id: string +} + +export type DeleteRowRequest = DeleteRows | DeleteRow + +export interface ValidateRowRequest extends Row {} +export interface ValidateRowResponse { + valid: boolean + errors: Record +} From 6494962c1ad47e52c5046dc3fd24a752ca579952 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 18:06:09 +0000 Subject: [PATCH 155/192] Screens and some slightly changes to Table API typing. --- packages/backend-core/src/security/roles.ts | 5 ++++- packages/server/src/api/controllers/screen.ts | 13 +++++++++---- packages/server/src/api/controllers/script.ts | 4 ---- .../server/src/api/controllers/table/index.ts | 15 +++++++++------ packages/server/src/api/routes/script.ts | 10 ---------- packages/server/src/sdk/app/tables/getters.ts | 6 ++++-- .../server/src/tests/utilities/api/table.ts | 19 +++++++++++-------- packages/types/src/api/web/app/screen.ts | 11 ++++++++++- packages/types/src/api/web/app/table.ts | 18 ++++++++---------- 9 files changed, 55 insertions(+), 46 deletions(-) delete mode 100644 packages/server/src/api/routes/script.ts diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 4076be93a0..2c424a4900 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -592,7 +592,10 @@ export class AccessController { ) } - async checkScreensAccess(screens: Screen[], userRoleId: string) { + async checkScreensAccess( + screens: Screen[], + userRoleId: string + ): Promise { let accessibleScreens = [] // don't want to handle this with Promise.all as this would mean all custom roles would be // retrieved at same time, it is likely a custom role will be re-used and therefore want diff --git a/packages/server/src/api/controllers/screen.ts b/packages/server/src/api/controllers/screen.ts index ee8e0ff892..cccc0c4d77 100644 --- a/packages/server/src/api/controllers/screen.ts +++ b/packages/server/src/api/controllers/screen.ts @@ -10,13 +10,16 @@ import { updateAppPackage } from "./application" import { Plugin, ScreenProps, - BBContext, Screen, UserCtx, + FetchScreenResponse, + SaveScreenRequest, + SaveScreenResponse, + DeleteScreenResponse, } from "@budibase/types" import { builderSocket } from "../../websockets" -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { const db = context.getAppDB() const screens = ( @@ -37,7 +40,9 @@ export async function fetch(ctx: BBContext) { ) } -export async function save(ctx: UserCtx) { +export async function save( + ctx: UserCtx +) { const db = context.getAppDB() let screen = ctx.request.body @@ -107,7 +112,7 @@ export async function save(ctx: UserCtx) { builderSocket?.emitScreenUpdate(ctx, savedScreen) } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const id = ctx.params.screenId const screen = await db.get(id) diff --git a/packages/server/src/api/controllers/script.ts b/packages/server/src/api/controllers/script.ts index 0565b30f74..ac6ac24fcc 100644 --- a/packages/server/src/api/controllers/script.ts +++ b/packages/server/src/api/controllers/script.ts @@ -14,7 +14,3 @@ export async function execute(ctx: Ctx) { throw err } } - -export async function save(ctx: Ctx) { - ctx.throw(501, "Not currently implemented") -} diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 77c1f3923a..4b020290e9 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -19,17 +19,18 @@ import { EventType, FetchTablesResponse, FieldType, - MigrateRequest, - MigrateResponse, + MigrateTableRequest, + MigrateTableResponse, SaveTableRequest, SaveTableResponse, Table, - TableResponse, + FindTableResponse, TableSourceType, UserCtx, ValidateNewTableImportRequest, ValidateTableImportRequest, ValidateTableImportResponse, + DeleteTableResponse, } from "@budibase/types" import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" @@ -94,7 +95,7 @@ export async function fetch(ctx: UserCtx) { ctx.body = result } -export async function find(ctx: UserCtx) { +export async function find(ctx: UserCtx) { const tableId = ctx.params.tableId const table = await sdk.tables.getTable(tableId) @@ -137,7 +138,7 @@ export async function save(ctx: UserCtx) { builderSocket?.emitTableUpdate(ctx, cloneDeep(savedTable)) } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: UserCtx) { const appId = ctx.appId const tableId = ctx.params.tableId await sdk.rowActions.deleteAll(tableId) @@ -223,7 +224,9 @@ export async function validateExistingTableImport( } } -export async function migrate(ctx: UserCtx) { +export async function migrate( + ctx: UserCtx +) { const { oldColumn, newColumn } = ctx.request.body let tableId = ctx.params.tableId as string const table = await sdk.tables.getTable(tableId) diff --git a/packages/server/src/api/routes/script.ts b/packages/server/src/api/routes/script.ts deleted file mode 100644 index 3780142ab5..0000000000 --- a/packages/server/src/api/routes/script.ts +++ /dev/null @@ -1,10 +0,0 @@ -import Router from "@koa/router" -import * as controller from "../controllers/script" -import authorized from "../../middleware/authorized" -import { permissions } from "@budibase/backend-core" - -const router: Router = new Router() - -router.post("/api/script", authorized(permissions.BUILDER), controller.save) - -export default router diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 79091ea660..f32150c63d 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -9,7 +9,7 @@ import { Database, INTERNAL_TABLE_SOURCE_ID, Table, - TableResponse, + FindTableResponse, TableSourceType, TableViewsResponse, } from "@budibase/types" @@ -173,7 +173,9 @@ export async function getTables(tableIds: string[]): Promise { return await processTables(tables) } -export async function enrichViewSchemas(table: Table): Promise { +export async function enrichViewSchemas( + table: Table +): Promise { const views = [] for (const view of Object.values(table.views ?? [])) { if (sdk.views.isV2(view)) { diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts index baaf890b52..86c9002c03 100644 --- a/packages/server/src/tests/utilities/api/table.ts +++ b/packages/server/src/tests/utilities/api/table.ts @@ -3,8 +3,8 @@ import { BulkImportResponse, CsvToJsonRequest, CsvToJsonResponse, - MigrateRequest, - MigrateResponse, + MigrateTableRequest, + MigrateTableResponse, SaveTableRequest, SaveTableResponse, Table, @@ -38,13 +38,16 @@ export class TableAPI extends TestAPI { migrate = async ( tableId: string, - data: MigrateRequest, + data: MigrateTableRequest, expectations?: Expectations - ): Promise => { - return await this._post(`/api/tables/${tableId}/migrate`, { - body: data, - expectations, - }) + ): Promise => { + return await this._post( + `/api/tables/${tableId}/migrate`, + { + body: data, + expectations, + } + ) } import = async ( diff --git a/packages/types/src/api/web/app/screen.ts b/packages/types/src/api/web/app/screen.ts index fc9eb63975..372d1ba2ea 100644 --- a/packages/types/src/api/web/app/screen.ts +++ b/packages/types/src/api/web/app/screen.ts @@ -1,4 +1,4 @@ -import { ScreenRoutingJson } from "../../../documents" +import { ScreenRoutingJson, Screen } from "../../../documents" export interface FetchScreenRoutingResponse { routes: ScreenRoutingJson @@ -6,3 +6,12 @@ export interface FetchScreenRoutingResponse { export interface FetchClientScreenRoutingResponse extends FetchScreenRoutingResponse {} + +export type FetchScreenResponse = Screen[] + +export interface SaveScreenRequest extends Screen {} +export interface SaveScreenResponse extends Screen {} + +export interface DeleteScreenResponse { + message: string +} diff --git a/packages/types/src/api/web/app/table.ts b/packages/types/src/api/web/app/table.ts index bb9329f6c1..f5d0d71560 100644 --- a/packages/types/src/api/web/app/table.ts +++ b/packages/types/src/api/web/app/table.ts @@ -3,33 +3,30 @@ import { ViewV2Enriched } from "../../../sdk" export type TableViewsResponse = { [key: string]: View | ViewV2Enriched } -export interface TableResponse extends Table { +export interface FindTableResponse extends Table { views?: TableViewsResponse } -export type FetchTablesResponse = TableResponse[] +export type FetchTablesResponse = FindTableResponse[] export interface SaveTableRequest extends TableRequest { rows?: Row[] } - export type SaveTableResponse = Table export interface BulkImportRequest { rows: Row[] identifierFields?: Array } - export interface BulkImportResponse { message: string } -export interface MigrateRequest { +export interface MigrateTableRequest { oldColumn: string newColumn: string } - -export interface MigrateResponse { +export interface MigrateTableResponse { message: string } @@ -37,12 +34,10 @@ export interface ValidateNewTableImportRequest { rows: Row[] schema: TableSchema } - export interface ValidateTableImportRequest { tableId?: string rows: Row[] } - export interface ValidateTableImportResponse { schemaValidation: { [field: string]: boolean @@ -55,5 +50,8 @@ export interface ValidateTableImportResponse { export interface CsvToJsonRequest { csvString: string } - export type CsvToJsonResponse = any[] + +export interface DeleteTableResponse { + message: string +} From 0b8f2c111d998ce10341742dafd517c93a05d5bd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 3 Dec 2024 18:07:21 +0000 Subject: [PATCH 156/192] Linting. --- packages/server/src/api/controllers/layout.ts | 1 - packages/server/src/automations/steps/executeQuery.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts index 447b437a23..a0eee000aa 100644 --- a/packages/server/src/api/controllers/layout.ts +++ b/packages/server/src/api/controllers/layout.ts @@ -2,7 +2,6 @@ import { EMPTY_LAYOUT } from "../../constants/layouts" import { generateLayoutID, getScreenParams } from "../../db/utils" import { events, context } from "@budibase/backend-core" import { - BBContext, DeleteLayoutResponse, Layout, SaveLayoutRequest, diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index 095c91a44d..7af540093f 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -12,7 +12,6 @@ import { ExecuteQueryStepInputs, ExecuteQueryStepOutputs, } from "@budibase/types" -import { executeV2AsAutomation } from "../../api/controllers/query" export const definition: AutomationStepDefinition = { name: "External Data Connector", From 92e5667b8176b935a786c0749955d040a53550cd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Dec 2024 18:55:06 +0000 Subject: [PATCH 157/192] prevent empty snippets being saved, don't crash for old empty snippets --- .../src/components/common/bindings/SnippetDrawer.svelte | 8 ++------ .../components/common/bindings/SnippetSidePanel.svelte | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/common/bindings/SnippetDrawer.svelte b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte index d6b6f92b17..d8e56b059a 100644 --- a/packages/builder/src/components/common/bindings/SnippetDrawer.svelte +++ b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte @@ -63,7 +63,7 @@ if (!name?.length) { return "Name is required" } - if (snippets.some(snippet => snippet.name === name)) { + if (!snippet?.name && snippets.some(snippet => snippet.name === name)) { return "That name is already in use" } if (firstCharNumberRegex.test(name)) { @@ -106,11 +106,7 @@ Delete {/if} - diff --git a/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte index c68699fc0f..2cc27b91cd 100644 --- a/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/SnippetSidePanel.svelte @@ -186,7 +186,7 @@
{#key hoveredSnippet} From 82a1165077d259f39847eadbeab6f66b9f6c4cd4 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 3 Dec 2024 19:01:00 +0000 Subject: [PATCH 158/192] Bump version to 3.2.23 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index e67812f37b..13bf382d84 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.22", + "version": "3.2.23", "npmClient": "yarn", "concurrency": 20, "command": { From 7fcff8a628bb1bd414e962f46ac504cbd57f785e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 4 Dec 2024 10:15:22 +0000 Subject: [PATCH 159/192] rely on llm status rather than initialised variable --- packages/pro | 2 +- packages/server/src/utilities/rowProcessor/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/pro b/packages/pro index e60f4b1b36..0d7fa31d4c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit e60f4b1b364fd49d2bb082f298757f83cb2032f0 +Subproject commit 0d7fa31d4c4019690e2200323421025cdc74b89e diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 9dbeb8ebb2..3728401ab8 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -126,8 +126,8 @@ export async function processAIColumns( const numRows = Array.isArray(inputRows) ? inputRows.length : 1 span?.addTags({ table_id: table._id, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini") - if (rows && llm.initialised) { + const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini") + if (rows && llmWrapper.llm) { // Ensure we have snippet context await context.ensureSnippetContext() From e119e310efe30fa2fdaf8f5516206578cfe8385b Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 4 Dec 2024 10:19:57 +0000 Subject: [PATCH 160/192] pro ref --- packages/pro | 2 +- packages/server/src/automations/steps/openai.ts | 8 ++++---- packages/server/src/utilities/rowProcessor/utils.ts | 8 +++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/pro b/packages/pro index 0d7fa31d4c..7b8789efd9 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 0d7fa31d4c4019690e2200323421025cdc74b89e +Subproject commit 7b8789efd940d9f8e5be9927243b19f07361c445 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 6f9adec1dc..8464c49763 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -106,13 +106,13 @@ export async function run({ (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) && (await pro.features.isBudibaseAIEnabled()) - let llm + let llmWrapper if (budibaseAIEnabled || customConfigsEnabled) { - llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) + llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) } - response = llm?.initialised - ? await llm.run(inputs.prompt) + response = llmWrapper?.llm + ? await llmWrapper.run(inputs.prompt) : await legacyOpenAIPrompt(inputs) return { diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 3728401ab8..09d3324ded 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -126,7 +126,9 @@ export async function processAIColumns( const numRows = Array.isArray(inputRows) ? inputRows.length : 1 span?.addTags({ table_id: table._id, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini") + const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( + "gpt-4o-mini" + ) if (rows && llmWrapper.llm) { // Ensure we have snippet context await context.ensureSnippetContext() @@ -151,14 +153,14 @@ export async function processAIColumns( } } - const prompt = llm.buildPromptFromAIOperation({ + const prompt = llmWrapper.buildPromptFromAIOperation({ schema: aiSchema, row, }) return tracer.trace("processAIColumn", {}, async span => { span?.addTags({ table_id: table._id, column }) - const llmResponse = await llm.run(prompt!) + const llmResponse = await llmWrapper.run(prompt!) return { ...row, [column]: llmResponse, From 46bd790b3a2950ff3b1ace16354e4fe3c3ca2008 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 4 Dec 2024 10:35:04 +0000 Subject: [PATCH 161/192] update tests --- packages/server/src/api/routes/tests/row.spec.ts | 2 +- packages/server/src/automations/steps/openai.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 1f4c4bc7cb..fb728a3fea 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -48,7 +48,7 @@ jest.mock("@budibase/pro", () => ({ ai: { LargeLanguageModel: { forCurrentTenant: async () => ({ - initialised: true, + llm: {}, run: jest.fn(() => `Mock LLM Response`), buildPromptFromAIOperation: jest.fn(), }), diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 8464c49763..48eaa93057 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -108,7 +108,9 @@ export async function run({ let llmWrapper if (budibaseAIEnabled || customConfigsEnabled) { - llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) + llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( + inputs.model + ) } response = llmWrapper?.llm From 48a367c2c2affa98e231132abaeb47716c139b1c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 4 Dec 2024 10:43:44 +0000 Subject: [PATCH 162/192] fix openai automation tests --- packages/server/src/automations/tests/openai.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 9f2bc50599..1985465fc0 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -27,7 +27,7 @@ jest.mock("@budibase/pro", () => ({ ai: { LargeLanguageModel: { forCurrentTenant: jest.fn().mockImplementation(() => ({ - initialised: true, + llm: {}, init: jest.fn(), run: jest.fn(), })), From 3d8e15abc3a5840f82816385ba5257a96316e57d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 4 Dec 2024 10:59:31 +0000 Subject: [PATCH 163/192] more usages of initialised --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- packages/server/src/utilities/rowProcessor/tests/utils.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c66197334e..dd1221d6fb 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -52,7 +52,7 @@ jest.mock("@budibase/pro", () => ({ ai: { LargeLanguageModel: { forCurrentTenant: async () => ({ - initialised: true, + llm: {}, run: jest.fn(() => `Mock LLM Response`), buildPromptFromAIOperation: jest.fn(), }), diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts index 3638b62892..fa674fcc52 100644 --- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts @@ -18,7 +18,7 @@ jest.mock("@budibase/pro", () => ({ ai: { LargeLanguageModel: { forCurrentTenant: async () => ({ - initialised: true, + llm: {}, run: jest.fn(() => "response from LLM"), buildPromptFromAIOperation: buildPromptMock, }), From 8d897ca434e1e387a5e40ebf194202c2f4fe9305 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 4 Dec 2024 11:09:46 +0000 Subject: [PATCH 164/192] Bump version to 3.2.24 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 13bf382d84..e960647b85 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.2.23", + "version": "3.2.24", "npmClient": "yarn", "concurrency": 20, "command": { From da81381d62e08af19b1635afe1233dd6a3feb864 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Dec 2024 13:48:30 +0000 Subject: [PATCH 165/192] PR review comments. --- packages/server/src/api/controllers/integration.ts | 2 +- packages/server/src/api/controllers/plugin/file.ts | 4 ++-- packages/server/src/sdk/plugins/plugins.ts | 4 ++-- packages/types/src/documents/global/plugin.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 9f770ad049..f26f8dcc52 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -26,7 +26,7 @@ export async function find(ctx: UserCtx) { } const integration = await getDefinition(ctx.params.type) if (!integration) { - ctx.throw(400, "Integration not found") + ctx.throw(404, "Integration not found") } ctx.body = integration } diff --git a/packages/server/src/api/controllers/plugin/file.ts b/packages/server/src/api/controllers/plugin/file.ts index 0f40998617..4b1f098939 100644 --- a/packages/server/src/api/controllers/plugin/file.ts +++ b/packages/server/src/api/controllers/plugin/file.ts @@ -3,9 +3,9 @@ import { getPluginMetadata, extractTarball, } from "../../../utilities/fileSystem" -import { FileType } from "@budibase/types" +import { KoaFile } from "@budibase/types" -export async function fileUpload(file: FileType) { +export async function fileUpload(file: KoaFile) { if (!file.name || !file.path) { throw new Error("File is not valid - cannot upload.") } diff --git a/packages/server/src/sdk/plugins/plugins.ts b/packages/server/src/sdk/plugins/plugins.ts index 650065b40c..63f2f22cd9 100644 --- a/packages/server/src/sdk/plugins/plugins.ts +++ b/packages/server/src/sdk/plugins/plugins.ts @@ -1,4 +1,4 @@ -import { FileType, Plugin, PluginSource, PluginType } from "@budibase/types" +import { KoaFile, Plugin, PluginSource, PluginType } from "@budibase/types" import { db as dbCore, objectStore, @@ -26,7 +26,7 @@ export async function fetch(type?: PluginType): Promise { } } -export async function processUploaded(plugin: FileType, source?: PluginSource) { +export async function processUploaded(plugin: KoaFile, source?: PluginSource) { const { metadata, directory } = await fileUpload(plugin) pluginCore.validate(metadata?.schema) diff --git a/packages/types/src/documents/global/plugin.ts b/packages/types/src/documents/global/plugin.ts index 5a7c701ae5..eeddc04e58 100644 --- a/packages/types/src/documents/global/plugin.ts +++ b/packages/types/src/documents/global/plugin.ts @@ -12,7 +12,7 @@ export enum PluginSource { URL = "URL", FILE = "File Upload", } -export interface FileType { +export interface KoaFile { path: string | null name: string | null } From fdb328aaa1ddf28a76261a3a0c1cf87856d7f45b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Dec 2024 14:27:09 +0000 Subject: [PATCH 166/192] Moving some types around - make sure app types are all correctly filed away. --- .../server/src/api/controllers/templates.ts | 12 +++- .../src/api/web/{ => app}/application.ts | 4 +- packages/types/src/api/web/app/automation.ts | 56 ++++++++++++++++++ .../types/src/api/web/{ => app}/deployment.ts | 2 +- packages/types/src/api/web/app/index.ts | 4 ++ .../types/src/api/web/{ => app}/layout.ts | 2 +- packages/types/src/api/web/{ => app}/role.ts | 4 +- packages/types/src/api/web/automation.ts | 58 ------------------- packages/types/src/api/web/index.ts | 6 +- packages/types/src/api/web/template.ts | 5 ++ 10 files changed, 81 insertions(+), 72 deletions(-) rename packages/types/src/api/web/{ => app}/application.ts (93%) rename packages/types/src/api/web/{ => app}/deployment.ts (78%) rename packages/types/src/api/web/{ => app}/layout.ts (79%) rename packages/types/src/api/web/{ => app}/role.ts (79%) delete mode 100644 packages/types/src/api/web/automation.ts create mode 100644 packages/types/src/api/web/template.ts diff --git a/packages/server/src/api/controllers/templates.ts b/packages/server/src/api/controllers/templates.ts index 6774fa2766..8008b63fd3 100644 --- a/packages/server/src/api/controllers/templates.ts +++ b/packages/server/src/api/controllers/templates.ts @@ -1,13 +1,17 @@ import nodeFetch from "node-fetch" import { downloadTemplate as dlTemplate } from "../../utilities/fileSystem" import env from "../../environment" -import { BBContext } from "@budibase/types" +import { + DownloadTemplateResponse, + FetchTemplateResponse, + UserCtx, +} from "@budibase/types" // development flag, can be used to test against templates exported locally const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com" -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { let type = env.TEMPLATE_REPOSITORY let response, error = false @@ -32,7 +36,9 @@ export async function fetch(ctx: BBContext) { // can't currently test this, have to ignore from coverage /* istanbul ignore next */ -export async function downloadTemplate(ctx: BBContext) { +export async function downloadTemplate( + ctx: UserCtx +) { const { type, name } = ctx.params await dlTemplate(type, name) diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/app/application.ts similarity index 93% rename from packages/types/src/api/web/application.ts rename to packages/types/src/api/web/app/application.ts index ed999ee574..bc00ade217 100644 --- a/packages/types/src/api/web/application.ts +++ b/packages/types/src/api/web/app/application.ts @@ -1,5 +1,5 @@ -import type { PlanType } from "../../sdk" -import type { Layout, App, Screen } from "../../documents" +import type { PlanType } from "../../../sdk" +import type { Layout, App, Screen } from "../../../documents" import { ReadStream } from "fs" export interface SyncAppResponse { diff --git a/packages/types/src/api/web/app/automation.ts b/packages/types/src/api/web/app/automation.ts index c10f19c88d..40f69fc467 100644 --- a/packages/types/src/api/web/app/automation.ts +++ b/packages/types/src/api/web/app/automation.ts @@ -1,9 +1,14 @@ import { + Automation, AutomationActionStepId, + AutomationLogPage, + AutomationStatus, AutomationStepDefinition, AutomationTriggerDefinition, AutomationTriggerStepId, + Row, } from "../../../documents" +import { DocumentDestroyResponse } from "@budibase/nano" export type GetAutomationTriggerDefinitionsResponse = Record< keyof typeof AutomationTriggerStepId, @@ -19,3 +24,54 @@ export interface GetAutomationStepDefinitionsResponse { trigger: GetAutomationTriggerDefinitionsResponse action: GetAutomationActionDefinitionsResponse } + +export interface DeleteAutomationResponse extends DocumentDestroyResponse {} + +export interface FetchAutomationResponse { + automations: Automation[] +} + +export interface FindAutomationResponse extends Automation {} + +export interface UpdateAutomationRequest extends Automation {} +export interface UpdateAutomationResponse { + message: string + automation: Automation +} + +export interface CreateAutomationRequest extends Automation {} +export interface CreateAutomationResponse { + message: string + automation: Automation +} + +export interface SearchAutomationLogsRequest { + startDate?: string + status?: AutomationStatus + automationId?: string + page?: string +} +export interface SearchAutomationLogsResponse extends AutomationLogPage {} + +export interface ClearAutomationLogRequest { + automationId: string + appId: string +} +export interface ClearAutomationLogResponse { + message: string +} + +export interface TriggerAutomationRequest { + fields: Record + // time in seconds + timeout: number +} +export type TriggerAutomationResponse = Record | undefined + +export interface TestAutomationRequest { + id?: string + revision?: string + fields: Record + row?: Row +} +export interface TestAutomationResponse {} diff --git a/packages/types/src/api/web/deployment.ts b/packages/types/src/api/web/app/deployment.ts similarity index 78% rename from packages/types/src/api/web/deployment.ts rename to packages/types/src/api/web/app/deployment.ts index f5ed9242b1..eed3f6eb77 100644 --- a/packages/types/src/api/web/deployment.ts +++ b/packages/types/src/api/web/app/deployment.ts @@ -1,4 +1,4 @@ -import { DeploymentDoc, DeploymentStatus } from "../../documents" +import { DeploymentDoc, DeploymentStatus } from "../../../documents" export interface PublishAppResponse extends DeploymentDoc {} diff --git a/packages/types/src/api/web/app/index.ts b/packages/types/src/api/web/app/index.ts index d130d19a42..90dfd7833e 100644 --- a/packages/types/src/api/web/app/index.ts +++ b/packages/types/src/api/web/app/index.ts @@ -13,3 +13,7 @@ export * from "./integration" export * from "./metadata" export * from "./query" export * from "./screen" +export * from "./application" +export * from "./layout" +export * from "./deployment" +export * from "./role" diff --git a/packages/types/src/api/web/layout.ts b/packages/types/src/api/web/app/layout.ts similarity index 79% rename from packages/types/src/api/web/layout.ts rename to packages/types/src/api/web/app/layout.ts index 45f7fb9d0a..3067a1f49f 100644 --- a/packages/types/src/api/web/layout.ts +++ b/packages/types/src/api/web/app/layout.ts @@ -1,4 +1,4 @@ -import { Layout } from "../../documents" +import { Layout } from "../../../documents" export interface SaveLayoutRequest extends Layout {} diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/app/role.ts similarity index 79% rename from packages/types/src/api/web/role.ts rename to packages/types/src/api/web/app/role.ts index eeedaea163..f23b2fd232 100644 --- a/packages/types/src/api/web/role.ts +++ b/packages/types/src/api/web/app/role.ts @@ -1,5 +1,5 @@ -import { Role, RoleUIMetadata } from "../../documents" -import { PermissionLevel, BuiltinPermissionID } from "../../sdk" +import { Role, RoleUIMetadata } from "../../../documents" +import { PermissionLevel, BuiltinPermissionID } from "../../../sdk" export interface SaveRoleRequest { _id?: string diff --git a/packages/types/src/api/web/automation.ts b/packages/types/src/api/web/automation.ts deleted file mode 100644 index 0f0699939e..0000000000 --- a/packages/types/src/api/web/automation.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { DocumentDestroyResponse } from "@budibase/nano" -import { - Automation, - AutomationLogPage, - AutomationStatus, - Row, -} from "../../documents" - -export interface DeleteAutomationResponse extends DocumentDestroyResponse {} - -export interface FetchAutomationResponse { - automations: Automation[] -} - -export interface FindAutomationResponse extends Automation {} - -export interface UpdateAutomationRequest extends Automation {} -export interface UpdateAutomationResponse { - message: string - automation: Automation -} - -export interface CreateAutomationRequest extends Automation {} -export interface CreateAutomationResponse { - message: string - automation: Automation -} - -export interface SearchAutomationLogsRequest { - startDate?: string - status?: AutomationStatus - automationId?: string - page?: string -} -export interface SearchAutomationLogsResponse extends AutomationLogPage {} - -export interface ClearAutomationLogRequest { - automationId: string - appId: string -} -export interface ClearAutomationLogResponse { - message: string -} - -export interface TriggerAutomationRequest { - fields: Record - // time in seconds - timeout: number -} -export type TriggerAutomationResponse = Record | undefined - -export interface TestAutomationRequest { - id?: string - revision?: string - fields: Record - row?: Row -} -export interface TestAutomationResponse {} diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index dd2ee3beae..9fcfb77f4f 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -1,4 +1,3 @@ -export * from "./application" export * from "./analytics" export * from "./auth" export * from "./user" @@ -11,10 +10,7 @@ export * from "./global" export * from "./pagination" export * from "./searchFilter" export * from "./cookies" -export * from "./automation" -export * from "./layout" -export * from "./role" export * from "./plugins" export * from "./apikeys" -export * from "./deployment" export * from "./dev" +export * from "./template" diff --git a/packages/types/src/api/web/template.ts b/packages/types/src/api/web/template.ts new file mode 100644 index 0000000000..bd5fdf1cb0 --- /dev/null +++ b/packages/types/src/api/web/template.ts @@ -0,0 +1,5 @@ +export interface FetchTemplateResponse {} + +export interface DownloadTemplateResponse { + message: string +} From 29d912e24eac78f5c20a7f98f81ac6aee930339c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Dec 2024 14:32:27 +0000 Subject: [PATCH 167/192] Template API typing. --- packages/types/src/api/web/template.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/types/src/api/web/template.ts b/packages/types/src/api/web/template.ts index bd5fdf1cb0..34b846b7ef 100644 --- a/packages/types/src/api/web/template.ts +++ b/packages/types/src/api/web/template.ts @@ -1,4 +1,20 @@ -export interface FetchTemplateResponse {} +export enum TemplateType { + APP = "app", +} + +export interface Template { + background: string + icon: string + category: string + description: string + name: string + url: string + type: TemplateType + key: string + image: string +} + +export type FetchTemplateResponse = Template[] export interface DownloadTemplateResponse { message: string From a7a2a56044717eaeb054ca767639269ad4c20c05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Dec 2024 15:00:05 +0000 Subject: [PATCH 168/192] User metadata API typing. --- packages/server/src/api/controllers/user.ts | 23 ++++++++++++++------- packages/types/src/api/web/app/user.ts | 20 ++++++++++++++++-- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index 108e29fd3d..d8ce37edb4 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -7,19 +7,24 @@ import { FetchUserMetadataResponse, FindUserMetadataResponse, Flags, - SetFlagRequest, + SetUserFlagRequest, + UpdateSelfMetadataRequest, + UpdateSelfMetadataResponse, + UpdateUserMetadataResponse, + UpdateUserMetadataRequest, UserCtx, - UserMetadata, + DeleteUserMetadataResponse, + SetUserFlagResponse, + GetUserFlagsResponse, } from "@budibase/types" import sdk from "../../sdk" -import { DocumentInsertResponse } from "@budibase/nano" export async function fetchMetadata(ctx: Ctx) { ctx.body = await sdk.users.fetchMetadata() } export async function updateSelfMetadata( - ctx: UserCtx + ctx: UserCtx ) { // overwrite the ID with current users ctx.request.body._id = ctx.user?._id @@ -31,7 +36,7 @@ export async function updateSelfMetadata( } export async function updateMetadata( - ctx: UserCtx + ctx: UserCtx ) { const db = context.getAppDB() const user = ctx.request.body @@ -44,7 +49,9 @@ export async function updateMetadata( ctx.body = await db.put(metadata) } -export async function destroyMetadata(ctx: UserCtx) { +export async function destroyMetadata( + ctx: UserCtx +) { const db = context.getAppDB() try { const dbUser = await sdk.users.get(ctx.params.id) @@ -64,7 +71,7 @@ export async function findMetadata( } export async function setFlag( - ctx: UserCtx + ctx: UserCtx ) { const userId = ctx.user?._id const { flag, value } = ctx.request.body @@ -84,7 +91,7 @@ export async function setFlag( ctx.body = { message: "Flag set successfully" } } -export async function getFlags(ctx: UserCtx) { +export async function getFlags(ctx: UserCtx) { const userId = ctx.user?._id const docId = generateUserFlagID(userId!) const db = context.getAppDB() diff --git a/packages/types/src/api/web/app/user.ts b/packages/types/src/api/web/app/user.ts index f5f2049724..ebdc43ea27 100644 --- a/packages/types/src/api/web/app/user.ts +++ b/packages/types/src/api/web/app/user.ts @@ -1,11 +1,27 @@ -import { ContextUserMetadata } from "../../../" +import { ContextUserMetadata, Flags, UserMetadata } from "../../../" +import { DocumentInsertResponse } from "@budibase/nano" export type FetchUserMetadataResponse = ContextUserMetadata[] export type FindUserMetadataResponse = ContextUserMetadata -export interface SetFlagRequest { +export interface SetUserFlagRequest { flag: string value: any } +export interface SetUserFlagResponse { + message: string +} + +export interface GetUserFlagsResponse extends Flags {} export type AppSelfResponse = ContextUserMetadata | {} + +export interface UpdateSelfMetadataRequest extends UserMetadata {} +export interface UpdateSelfMetadataResponse extends DocumentInsertResponse {} + +export interface UpdateUserMetadataRequest extends UserMetadata {} +export interface UpdateUserMetadataResponse extends DocumentInsertResponse {} + +export interface DeleteUserMetadataResponse { + message: string +} From aa2d6779cde3622ca3089e0b8787fad907ba139f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Dec 2024 15:02:16 +0000 Subject: [PATCH 169/192] Small view API typing update. --- packages/server/src/api/controllers/view/viewsV2.ts | 10 ++++++---- packages/types/src/api/web/app/view.ts | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 986764a697..c99b79d8fa 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -4,7 +4,6 @@ import { Ctx, RequiredKeys, UpdateViewRequest, - ViewResponse, ViewResponseEnriched, ViewV2, BasicViewFieldMetadata, @@ -15,6 +14,9 @@ import { ViewFetchResponseEnriched, CountDistinctCalculationFieldMetadata, CountCalculationFieldMetadata, + CreateViewResponse, + UpdateViewResponse, + DeleteViewResponse, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" @@ -132,7 +134,7 @@ export async function fetch(ctx: Ctx) { } } -export async function create(ctx: Ctx) { +export async function create(ctx: Ctx) { const view = ctx.request.body const { tableId } = view @@ -159,7 +161,7 @@ export async function create(ctx: Ctx) { gridSocket?.emitViewUpdate(ctx, result) } -export async function update(ctx: Ctx) { +export async function update(ctx: Ctx) { const view = ctx.request.body if (view.version !== 2) { @@ -196,7 +198,7 @@ export async function update(ctx: Ctx) { gridSocket?.emitViewUpdate(ctx, result) } -export async function remove(ctx: Ctx) { +export async function remove(ctx: Ctx) { const { viewId } = ctx.params const view = await sdk.views.remove(viewId) diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index 2560f7507f..8ce7d5ca9e 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -14,5 +14,7 @@ export interface ViewFetchResponseEnriched { } export interface CreateViewRequest extends Omit {} +export interface CreateViewResponse extends ViewResponse {} export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewResponse extends ViewResponse {} From bc8cd6723d0757defa0982d67434e0e7c5f8916e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 4 Dec 2024 15:54:07 +0000 Subject: [PATCH 170/192] Add automation survey popup --- .../[application]/automation/_layout.svelte | 74 ++++++++++++++++++- 1 file changed, 70 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte index acc1b8e2d9..cc2746306c 100644 --- a/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte @@ -1,5 +1,5 @@