From bb4209cd670b142ab042168f43d8f287eaff7762 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Fri, 26 May 2023 09:21:34 +0100 Subject: [PATCH 001/127] Basic modal put together. --- .../modals/DatasourceConfigModal.svelte | 22 ++- .../modals/FetchTablesModal.svelte | 21 +++ packages/server/package.json | 2 +- packages/worker/package.json | 2 +- yarn.lock | 127 ++++++------------ 5 files changed, 79 insertions(+), 95 deletions(-) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index e75109b1be..4c47720692 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -1,7 +1,14 @@ + + + + saveDatasource()} + onConfirm={() => + datasourcePlus ? saveDatasource() : fetchTablesModal.show()} onCancel={() => modal.show()} - confirmText={datasource.plus ? "Connect" : "Save and continue to query"} + confirmText={datasourcePlus ? "Connect" : "Save and continue to query"} cancelText="Back" - showSecondaryButton={datasource.plus} + showSecondaryButton={datasourcePlus} size="L" disabled={!isValid} > diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte new file mode 100644 index 0000000000..41a1f2d2d0 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte @@ -0,0 +1,21 @@ + + + prevModal.show()} + confirmText="Continue" + cancelText="Back" + size="L" + disabled={selected.length === 0} +> + SOME TABLES HERE + diff --git a/packages/server/package.json b/packages/server/package.json index 136eeb609b..2ec1395b71 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "0.0.1", "@budibase/client": "0.0.1", - "@budibase/pro": "develop", + "@budibase/pro": "0.0.1", "@budibase/shared-core": "0.0.1", "@budibase/string-templates": "0.0.1", "@budibase/types": "0.0.1", diff --git a/packages/worker/package.json b/packages/worker/package.json index dded20cf30..684bd8755b 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "0.0.1", - "@budibase/pro": "develop", + "@budibase/pro": "0.0.1", "@budibase/string-templates": "0.0.1", "@budibase/types": "0.0.1", "@koa/router": "8.0.8", diff --git a/yarn.lock b/yarn.lock index 33225932ea..0d9d795b8a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1728,47 +1728,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.6.19-alpha.4.tgz#525dd7ba6c5db4404cf00d1165f79d34a1093826" - integrity sha512-1pfOr+J9xYawedVmvqpQ4b/8C2SQP4cKhFmSz5uErM2SCgbRj+JuzOUTPNX0vzAXPvat/kEegt79xThummDvhA== - dependencies: - "@budibase/nano" "10.1.2" - "@budibase/pouchdb-replication-stream" "1.2.10" - "@budibase/types" "2.6.19-alpha.4" - "@shopify/jest-koa-mocks" "5.0.1" - "@techpass/passport-openidconnect" "0.3.2" - aws-cloudfront-sign "2.2.0" - aws-sdk "2.1030.0" - bcrypt "5.0.1" - bcryptjs "2.4.3" - bull "4.10.1" - correlation-id "4.0.0" - dotenv "16.0.1" - emitter-listener "1.1.2" - ioredis "4.28.0" - joi "17.6.0" - jsonwebtoken "9.0.0" - koa-passport "4.1.4" - koa-pino-logger "4.0.0" - lodash "4.17.21" - lodash.isarguments "3.1.0" - node-fetch "2.6.7" - passport-google-oauth "2.0.0" - passport-jwt "4.0.0" - passport-local "1.0.0" - passport-oauth2-refresh "^2.1.0" - pino "8.11.0" - pino-http "8.3.3" - posthog-node "1.3.0" - pouchdb "7.3.0" - pouchdb-find "7.2.2" - redlock "4.2.0" - sanitize-s3-objectkey "0.0.1" - semver "7.3.7" - tar-fs "2.1.1" - uuid "8.3.2" - "@budibase/bbui@^0.9.139": version "0.9.190" resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.190.tgz#e1ec400ac90f556bfbc80fc23a04506f1585ea81" @@ -1869,32 +1828,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@develop": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.19-alpha.4.tgz#5d4c885ac9ac4ccfb2f8896961aca903c1178750" - integrity sha512-iu2QzV8Z77c00muBSK+NVsZdug3lLD0lR+vcKancGEz1PPE4yNIH7g8jB6i8h9agArbx9S2ICeHQqGb6nQaGHQ== - dependencies: - "@budibase/backend-core" "2.6.19-alpha.4" - "@budibase/shared-core" "2.6.19-alpha.4" - "@budibase/string-templates" "2.6.19-alpha.4" - "@budibase/types" "2.6.19-alpha.4" - "@koa/router" "8.0.8" - bull "4.10.1" - joi "17.6.0" - jsonwebtoken "8.5.1" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "^2.6.1" - scim-patch "^0.7.0" - scim2-parse-filter "^0.2.8" - -"@budibase/shared-core@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.6.19-alpha.4.tgz#dd22dd0a18ee4d6739b629f461e5caec90706282" - integrity sha512-ac6iWSsgz70OYbdA+QHPLpTnRbIZ4OecVc6Y7gnEZ78hZ4S5da8a+73jswuy0/t4YsiT/gjukjzjoihg3NemVg== - dependencies: - "@budibase/types" "2.6.19-alpha.4" - "@budibase/standard-components@^0.9.139": version "0.9.139" resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.139.tgz#cf8e2b759ae863e469e50272b3ca87f2827e66e3" @@ -1913,25 +1846,6 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.6.19-alpha.4.tgz#92ebd69a6841174b8af91f338c4754ca7c402707" - integrity sha512-KsH3NlQcJibRj98Q8zQ3KQHhfSIWPQfvR80MmBTIe05llEZGox4re4pQQUnlMafaUEyNNtIqVnbTJ1XP0LmFng== - dependencies: - "@budibase/handlebars-helpers" "^0.11.8" - dayjs "^1.10.4" - handlebars "^4.7.6" - handlebars-utils "^1.0.6" - lodash "^4.17.20" - vm2 "^3.9.15" - -"@budibase/types@2.6.19-alpha.4": - version "2.6.19-alpha.4" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.6.19-alpha.4.tgz#bcf81699329d3f8509e4b0a489211f35b6cfa7ce" - integrity sha512-qFsXHZTSigcfCv02aTZGsf17vBT/MC+zK9ky7WZVX4h0sJiE0li4A66/tMaSDz3/vQ7ToPRhJK/p+LOWA/oceg== - dependencies: - scim-patch "^0.7.0" - "@bull-board/api@3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af" @@ -3543,6 +3457,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/sourcemap-codec@^1.4.13": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + "@jridgewell/trace-mapping@0.3.9": version "0.3.9" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" @@ -4323,7 +4242,7 @@ dependencies: slash "^3.0.0" -"@rollup/plugin-commonjs@^16.0.0": +"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0": version "16.0.0" resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f" integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw== @@ -4406,6 +4325,22 @@ "@rollup/pluginutils" "^3.1.0" magic-string "^0.25.7" +"@rollup/plugin-replace@^5.0.2": + version "5.0.2" + resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.2.tgz#45f53501b16311feded2485e98419acb8448c61d" + integrity sha512-M9YXNekv/C/iHHK+cvORzfRYfPbq0RDD8r0G+bMiTXjNGKulPnCT9O3Ss46WfhI6ZOCgApOP7xAdmCQJ+U2LAA== + dependencies: + "@rollup/pluginutils" "^5.0.1" + magic-string "^0.27.0" + +"@rollup/plugin-typescript@8.3.0": + version "8.3.0" + resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-8.3.0.tgz#bc1077fa5897b980fc27e376c4e377882c63e68b" + integrity sha512-I5FpSvLbtAdwJ+naznv+B4sjXZUcIvLLceYpITAn7wAP8W0wqc5noLdGIp9HGVntNhRWXctwPYrSSFQxtl0FPA== + dependencies: + "@rollup/pluginutils" "^3.1.0" + resolve "^1.17.0" + "@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b" @@ -12433,7 +12368,7 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== @@ -17481,6 +17416,13 @@ magic-string@^0.26.2: dependencies: sourcemap-codec "^1.4.8" +magic-string@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3" + integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA== + dependencies: + "@jridgewell/sourcemap-codec" "^1.4.13" + make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -22307,6 +22249,13 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.5.0, rollup-pluginutils@^2.6.0, dependencies: estree-walker "^0.6.1" +rollup@2.45.2: + version "2.45.2" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.45.2.tgz#8fb85917c9f35605720e92328f3ccbfba6f78b48" + integrity sha512-kRRU7wXzFHUzBIv0GfoFFIN3m9oteY4uAsKllIpQDId5cfnkWF2J130l+27dzDju0E6MScKiV0ZM5Bw8m4blYQ== + optionalDependencies: + fsevents "~2.3.1" + rollup@^2.36.2, rollup@^2.44.0, rollup@^2.45.2, rollup@^2.79.1: version "2.79.1" resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7" @@ -24312,7 +24261,7 @@ timed-out@^4.0.1: resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA== -timekeeper@2.2.0: +timekeeper@2.2.0, timekeeper@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368" integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A== From a330e606bf1a8c655d2c0abbc1528d99b295f438 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jun 2023 15:17:06 +0100 Subject: [PATCH 002/127] Removing merge artifact. --- .../DatasourceNavigator/modals/DatasourceConfigModal.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 1696a1870b..52051fe134 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -75,7 +75,6 @@ title={`Connect to ${name}`} onConfirm={() => datasourcePlus ? saveDatasource() : fetchTablesModal.show()} - onCancel={() => modal.show()} confirmText={datasourcePlus ? "Connect" : "Save and continue to query"} cancelText="Back" showSecondaryButton={datasourcePlus} From 96f44c0a863569fbf8050f0bd460a38a1ff98488 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jun 2023 17:40:12 +0100 Subject: [PATCH 003/127] Reworking implementation to a single modal, that changes content rather than progressing to a new modal. --- .../modals/DatasourceConfigModal.svelte | 87 +++++++++++-------- .../modals/FetchTablesModal.svelte | 21 ----- packages/frontend-core/src/api/datasources.js | 11 +++ 3 files changed, 61 insertions(+), 58 deletions(-) delete mode 100644 packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 52051fe134..e9d23ff0c7 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -1,14 +1,7 @@ - - - - - datasourcePlus ? saveDatasource() : fetchTablesModal.show()} - confirmText={datasourcePlus ? "Connect" : "Save and continue to query"} - cancelText="Back" + {title} + onConfirm={() => nextStep()} + {confirmText} + cancelText={fetchTableStep ? "Cancel" : "Back"} showSecondaryButton={datasourcePlus} size="L" disabled={!isValid} > - - Connect your database to Budibase using the config below. - - - (isValid = e.detail)} - /> + {#if !fetchTableStep} + + Connect your database to Budibase using the config below. + + + (isValid = e.detail)} + /> + {:else} + Some stuff here + {/if} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte deleted file mode 100644 index 41a1f2d2d0..0000000000 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/FetchTablesModal.svelte +++ /dev/null @@ -1,21 +0,0 @@ - - - prevModal.show()} - confirmText="Continue" - cancelText="Back" - size="L" - disabled={selected.length === 0} -> - SOME TABLES HERE - diff --git a/packages/frontend-core/src/api/datasources.js b/packages/frontend-core/src/api/datasources.js index 16d19c512f..f7554f64b0 100644 --- a/packages/frontend-core/src/api/datasources.js +++ b/packages/frontend-core/src/api/datasources.js @@ -69,4 +69,15 @@ export const buildDatasourceEndpoints = API => ({ body: { datasource }, }) }, + + /** + * Fetch table names available within the datasource, for filtering out undesired tables + * @param datasource the datasource configuration to use for fetching tables + */ + fetchTablesForDatasource: async datasource => { + return await API.post({ + url: `/api/datasources/tables`, + body: { datasource }, + }) + }, }) From b57557760dd060db96091fa4cc2970e1bfb44525 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jun 2023 18:36:50 +0100 Subject: [PATCH 004/127] Updating fancy forms to be usable for data table fetching designs. --- .../bbui/src/FancyForm/FancyCheckbox.svelte | 14 +++++++++-- packages/bbui/src/FancyForm/FancyField.svelte | 4 +-- packages/bbui/src/FancyForm/FancyForm.svelte | 12 ++++++++- .../DatasourceNavigator/icons/index.js | 3 +++ .../modals/DatasourceConfigModal.svelte | 25 +++++++++++++++++-- 5 files changed, 51 insertions(+), 7 deletions(-) diff --git a/packages/bbui/src/FancyForm/FancyCheckbox.svelte b/packages/bbui/src/FancyForm/FancyCheckbox.svelte index 191cc79485..21fdb9d7a9 100644 --- a/packages/bbui/src/FancyForm/FancyCheckbox.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckbox.svelte @@ -8,6 +8,8 @@ export let disabled = false export let error = null export let validate = null + export let compact = false + export let noMaxWidth const dispatch = createEventDispatcher() @@ -21,7 +23,16 @@ } - + @@ -39,7 +50,6 @@ } .text { font-size: 15px; - line-height: 17px; overflow: hidden; text-overflow: ellipsis; display: -webkit-box; diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 0c99394599..10d847248d 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -61,7 +61,7 @@ From 7afbce696baf48b7ba89038ba150a5ea947544b9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 5 Jun 2023 18:18:08 +0100 Subject: [PATCH 005/127] Create listbox component --- packages/bbui/src/ListBox/Checkbox.svelte | 62 ++++++++++++++ packages/bbui/src/ListBox/ListBox.svelte | 99 +++++++++++++++++++++++ packages/bbui/src/index.js | 1 + 3 files changed, 162 insertions(+) create mode 100644 packages/bbui/src/ListBox/Checkbox.svelte create mode 100644 packages/bbui/src/ListBox/ListBox.svelte diff --git a/packages/bbui/src/ListBox/Checkbox.svelte b/packages/bbui/src/ListBox/Checkbox.svelte new file mode 100644 index 0000000000..3548e4868e --- /dev/null +++ b/packages/bbui/src/ListBox/Checkbox.svelte @@ -0,0 +1,62 @@ + + + + + diff --git a/packages/bbui/src/ListBox/ListBox.svelte b/packages/bbui/src/ListBox/ListBox.svelte new file mode 100644 index 0000000000..04b563160e --- /dev/null +++ b/packages/bbui/src/ListBox/ListBox.svelte @@ -0,0 +1,99 @@ + + +
+ {#if options && Array.isArray(options)} + {#if showSelectAll} +
+ +
+ {/if} + {#each options as option} +
+ +
+ {/each} + {/if} +
+ + diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index d26b938dd5..f1b2e58870 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -78,6 +78,7 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte export { default as Slider } from "./Form/Slider.svelte" export { default as Accordion } from "./Accordion/Accordion.svelte" export { default as File } from "./Form/File.svelte" +export { default as ListBox } from "./ListBox/ListBox.svelte" // Renderers export { default as BoldRenderer } from "./Table/BoldRenderer.svelte" From 1f428bd5c273b52f1e216b9e65bb33a59012e684 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 12:17:56 +0100 Subject: [PATCH 006/127] Fix test --- packages/server/src/api/controllers/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d21db8ad03..75b33083fb 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -29,7 +29,7 @@ import { } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" -import { setupCreationAuth as googleSetupCreationAuth } from "src/integrations/googlesheets" +import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" function getErrorTables(errors: any, errorType: string) { return Object.entries(errors) From b7fd069d00962233fe10f6fb89164e7c2e77aa59 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 13:49:11 +0100 Subject: [PATCH 007/127] Fixing endpoint to info from tables for datasource fetching. --- packages/frontend-core/src/api/datasources.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/api/datasources.js b/packages/frontend-core/src/api/datasources.js index f7554f64b0..fec897555c 100644 --- a/packages/frontend-core/src/api/datasources.js +++ b/packages/frontend-core/src/api/datasources.js @@ -74,9 +74,9 @@ export const buildDatasourceEndpoints = API => ({ * Fetch table names available within the datasource, for filtering out undesired tables * @param datasource the datasource configuration to use for fetching tables */ - fetchTablesForDatasource: async datasource => { + fetchInfoForDatasource: async datasource => { return await API.post({ - url: `/api/datasources/tables`, + url: `/api/datasources/info`, body: { datasource }, }) }, From 9e2096850f0ddb753051d3686ea6e811c12d82f2 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 6 Jun 2023 13:22:08 +0000 Subject: [PATCH 008/127] Bump version to 2.7.1-alpha.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 4e51d6b904..afa1ce1551 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.0", + "version": "2.7.1-alpha.0", "npmClient": "yarn", "packages": [ "packages/backend-core", From f9f51f9ecf073426f1487690814a94128635c5d8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 6 Jun 2023 13:58:06 +0000 Subject: [PATCH 009/127] Bump version to 2.7.1-alpha.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index afa1ce1551..21ffea6191 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.1-alpha.0", + "version": "2.7.1-alpha.1", "npmClient": "yarn", "packages": [ "packages/backend-core", From 0602f5f26f11e2ccca5e9e7e242a8d4e0536131f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 16:32:20 +0100 Subject: [PATCH 010/127] Updating fancy form components and building out first version of fancy checkbox group. --- .../bbui/src/FancyForm/FancyCheckbox.svelte | 8 +-- .../src/FancyForm/FancyCheckboxGroup.svelte | 53 +++++++++++++++++++ packages/bbui/src/FancyForm/FancyField.svelte | 14 ++++- packages/bbui/src/FancyForm/FancyForm.svelte | 3 +- packages/bbui/src/FancyForm/index.js | 1 + .../modals/DatasourceConfigModal.svelte | 11 +--- 6 files changed, 74 insertions(+), 16 deletions(-) create mode 100644 packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte diff --git a/packages/bbui/src/FancyForm/FancyCheckbox.svelte b/packages/bbui/src/FancyForm/FancyCheckbox.svelte index 21fdb9d7a9..363a827b1e 100644 --- a/packages/bbui/src/FancyForm/FancyCheckbox.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckbox.svelte @@ -8,8 +8,8 @@ export let disabled = false export let error = null export let validate = null - export let compact = false - export let noMaxWidth + export let compress = false + export let lighter = false const dispatch = createEventDispatcher() @@ -28,8 +28,8 @@ {value} {validate} {disabled} - {compact} - {noMaxWidth} + {compress} + {lighter} clickable on:click={onChange} > diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte new file mode 100644 index 0000000000..998f7afe94 --- /dev/null +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -0,0 +1,53 @@ + + +{#if options && Array.isArray(options)} + + {#if showSelectAll} +
+ +
+ {/if} + {#each options as option, i} + + {/each} + +{/if} + + diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 10d847248d..a4d453bb77 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -11,6 +11,8 @@ export let value export let ref export let autoHeight + export let compress + export let lighter const formContext = getContext("fancy-form") const id = Math.random() @@ -38,10 +40,12 @@
@@ -70,6 +74,12 @@ background 130ms ease-out; color: var(--spectrum-global-color-gray-800); } + .lighter { + background: var(--spectrum-global-color-gray-100) !important; + } + .compress { + margin-bottom: -1px; + } .fancy-field:hover { border-color: var(--spectrum-global-color-gray-400); } @@ -92,7 +102,7 @@ .content { position: relative; height: var(--fancy-field-height); - padding: 0 16px; + padding: 0 var(--fancy-field-padding); } .fancy-field.auto-height .content { height: auto; @@ -103,7 +113,7 @@ flex-direction: row; justify-content: flex-start; align-items: center; - gap: 16px; + gap: var(--fancy-field-padding); } .field { flex: 1 1 auto; diff --git a/packages/bbui/src/FancyForm/FancyForm.svelte b/packages/bbui/src/FancyForm/FancyForm.svelte index 0572bbd403..28bfbb4930 100644 --- a/packages/bbui/src/FancyForm/FancyForm.svelte +++ b/packages/bbui/src/FancyForm/FancyForm.svelte @@ -29,7 +29,8 @@ const styles = () => { let styleString = "" styleString += `--fancy-field-max-width: ${noMaxWidth ? "auto" : "400px"}` - styleString += `; --fancy-field-height: ${compact ? "40px" : "64px"}` + styleString += `; --fancy-field-height: ${compact ? "36px" : "64px"}` + styleString += `; --fancy-field-padding: ${compact ? "8px" : "16px"}` return styleString } diff --git a/packages/bbui/src/FancyForm/index.js b/packages/bbui/src/FancyForm/index.js index e105991c0f..33591a532c 100644 --- a/packages/bbui/src/FancyForm/index.js +++ b/packages/bbui/src/FancyForm/index.js @@ -4,4 +4,5 @@ export { default as FancySelect } from "./FancySelect.svelte" export { default as FancyButton } from "./FancyButton.svelte" export { default as FancyForm } from "./FancyForm.svelte" export { default as FancyButtonRadio } from "./FancyButtonRadio.svelte" +export { default as FancyCheckboxGroup } from "./FancyCheckboxGroup.svelte" export { default as ErrorMessage } from "./ErrorMessage.svelte" diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 103667848c..481dd29856 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -5,8 +5,7 @@ notifications, Body, Layout, - FancyForm, - FancyCheckbox, + FancyCheckboxGroup, } from "@budibase/bbui" import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte" import { IntegrationNames } from "constants/backend" @@ -111,13 +110,7 @@ /> {:else}
- - - - - - - +
{/if} From f6108998f9b54858e7c1ff9894b6b4040a264a51 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 16:52:06 +0100 Subject: [PATCH 011/127] Adding better selection control to fancy checkbox group. --- .../src/FancyForm/FancyCheckboxGroup.svelte | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte index 998f7afe94..58cf00db58 100644 --- a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -3,26 +3,35 @@ import FancyForm from "./FancyForm.svelte" export let options = [] - export let selected = [] + export let selected export let showSelectAll = true export let selectAllText = "Select all" - let allSelected = false - $: { - if (selected.length === options.length) { - allSelected = true - } else if (selected.length === 0) { - allSelected = false - } else { - allSelected = "partial" + let selectedBooleans = reset() + + $: selected = updateSelected(selectedBooleans) + $: console.log(selected) + $: allSelected = selected.length === options.length + + function reset() { + return Array(options.length).fill(true) + } + + function updateSelected(selectedArr) { + const array = [] + for (let [i, isSelected] of Object.entries(selectedArr)) { + if (isSelected) { + array.push(options[i]) + } } + return array } function toggleSelectAll() { if (allSelected === true) { - selected = [] + selectedBooleans = [] } else { - selected = [...options] + selectedBooleans = reset() } } @@ -41,7 +50,7 @@
{/if} {#each options as option, i} - + {/each} {/if} From cc60baa3d188526df93c4b4711befdebe4d22816 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 17:40:14 +0100 Subject: [PATCH 012/127] Some fixes for BBUI and updating postgres table name fetching to get uniques. --- .../src/FancyForm/FancyCheckboxGroup.svelte | 12 ++- packages/bbui/src/ListBox/Checkbox.svelte | 62 ------------ packages/bbui/src/ListBox/ListBox.svelte | 99 ------------------- packages/bbui/src/index.js | 1 - .../builder/src/builderStore/datasource.js | 8 +- .../modals/DatasourceConfigModal.svelte | 8 +- .../server/src/api/controllers/datasource.ts | 2 +- packages/server/src/integrations/postgres.ts | 3 +- 8 files changed, 23 insertions(+), 172 deletions(-) delete mode 100644 packages/bbui/src/ListBox/Checkbox.svelte delete mode 100644 packages/bbui/src/ListBox/ListBox.svelte diff --git a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte index 58cf00db58..be37994616 100644 --- a/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte +++ b/packages/bbui/src/FancyForm/FancyCheckboxGroup.svelte @@ -1,17 +1,19 @@ - - - - diff --git a/packages/bbui/src/ListBox/ListBox.svelte b/packages/bbui/src/ListBox/ListBox.svelte deleted file mode 100644 index 04b563160e..0000000000 --- a/packages/bbui/src/ListBox/ListBox.svelte +++ /dev/null @@ -1,99 +0,0 @@ - - -
- {#if options && Array.isArray(options)} - {#if showSelectAll} -
- -
- {/if} - {#each options as option} -
- -
- {/each} - {/if} -
- - diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index f1b2e58870..d26b938dd5 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -78,7 +78,6 @@ export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte export { default as Slider } from "./Form/Slider.svelte" export { default as Accordion } from "./Accordion/Accordion.svelte" export { default as File } from "./Form/File.svelte" -export { default as ListBox } from "./ListBox/ListBox.svelte" // Renderers export { default as BoldRenderer } from "./Table/BoldRenderer.svelte" diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 6509216a5b..10f1facb1c 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -41,6 +41,10 @@ export async function createRestDatasource(integration) { export async function validateDatasourceConfig(config) { const datasource = prepareData(config) - const resp = await API.validateDatasource(datasource) - return resp + return await API.validateDatasource(datasource) +} + +export async function getDatasourceInfo(config) { + const datasource = prepareData(config) + return await API.fetchInfoForDatasource(datasource) } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 481dd29856..626c61ea5f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -13,6 +13,7 @@ import { saveDatasource as save, validateDatasourceConfig, + getDatasourceInfo, } from "builderStore/datasource" import { DatasourceFeature } from "@budibase/types" @@ -22,7 +23,10 @@ let datasource = cloneDeep(integration) let isValid = false let fetchTableStep = false + let selectedTables = [] + let tableList = [] + $: console.log(selectedTables) $: name = IntegrationNames[datasource?.type] || datasource?.name || datasource?.type $: datasourcePlus = datasource?.plus @@ -78,6 +82,8 @@ } if (datasourcePlus) { notifications.success("Connected to datasource successfully.") + const info = await getDatasourceInfo(datasource) + tableList = info.tableNames fetchTableStep = true return false } else { @@ -110,7 +116,7 @@ /> {:else}
- +
{/if} diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 75b33083fb..4660d05b1e 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -174,7 +174,7 @@ export async function information( } const tableNames = await connector.getTableNames() ctx.body = { - tableNames, + tableNames: tableNames.sort(), } } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 47295716e6..33ecf45982 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -322,7 +322,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { await this.openConnection() const columnsResponse: { rows: PostgresColumn[] } = await this.client.query(this.COLUMNS_SQL) - return columnsResponse.rows.map(row => row.table_name) + const names = columnsResponse.rows.map(row => row.table_name) + return [...new Set(names)] } finally { await this.closeConnection() } From dab1db227154cd3ae72a89c73f9e17b3c8c911b5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 18:27:03 +0100 Subject: [PATCH 013/127] Updating datasource save API to allow specifying a filter for which tables to fetch and then linking this up with the new frontend. --- .../builder/src/builderStore/datasource.js | 5 +- .../modals/DatasourceConfigModal.svelte | 8 ++- .../builder/src/stores/backend/datasources.js | 6 ++- packages/frontend-core/src/api/datasources.js | 5 +- .../server/src/api/controllers/datasource.ts | 49 ++++++++++--------- packages/types/src/api/web/app/datasource.ts | 1 + 6 files changed, 46 insertions(+), 28 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 10f1facb1c..49250d2628 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -23,10 +23,11 @@ function prepareData(config) { return datasource } -export async function saveDatasource(config, skipFetch = false) { +export async function saveDatasource(config, { skipFetch, tablesFilter } = {}) { const datasource = prepareData(config) // Create datasource - const resp = await datasources.save(datasource, !skipFetch && datasource.plus) + const fetchSchema = !skipFetch && datasource.plus + const resp = await datasources.save(datasource, { fetchSchema, tablesFilter }) // update the tables incase datasource plus await tables.fetch() diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 626c61ea5f..3ab0b17840 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -62,7 +62,11 @@ if (!datasource.name) { datasource.name = name } - const resp = await save(datasource) + const opts = {} + if (datasourcePlus && selectedTables) { + opts.tablesFilter = selectedTables + } + const resp = await save(datasource, opts) $goto(`./datasource/${resp._id}`) notifications.success("Datasource created successfully.") } catch (err) { @@ -80,7 +84,7 @@ if (!connected) { return false } - if (datasourcePlus) { + if (datasourcePlus && !fetchTableStep) { notifications.success("Connected to datasource successfully.") const info = await getDatasourceInfo(datasource) tableList = info.tableNames diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index 0815f9d766..e774aae8c6 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -57,7 +57,10 @@ export function createDatasourcesStore() { return updateDatasource(response) } - const save = async (body, fetchSchema = false) => { + const save = async (body, { fetchSchema, tablesFilter } = {}) => { + if (fetchSchema == null) { + fetchSchema = false + } let response if (body._id) { response = await API.updateDatasource(body) @@ -65,6 +68,7 @@ export function createDatasourcesStore() { response = await API.createDatasource({ datasource: body, fetchSchema, + tablesFilter, }) } return updateDatasource(response) diff --git a/packages/frontend-core/src/api/datasources.js b/packages/frontend-core/src/api/datasources.js index fec897555c..1ba17cc610 100644 --- a/packages/frontend-core/src/api/datasources.js +++ b/packages/frontend-core/src/api/datasources.js @@ -26,13 +26,16 @@ export const buildDatasourceEndpoints = API => ({ * Creates a datasource * @param datasource the datasource to create * @param fetchSchema whether to fetch the schema or not + * @param tablesFilter a list of tables to actually fetch rather than simply + * all that are accessible. */ - createDatasource: async ({ datasource, fetchSchema }) => { + createDatasource: async ({ datasource, fetchSchema, tablesFilter }) => { return await API.post({ url: "/api/datasources", body: { datasource, fetchSchema, + tablesFilter, }, }) }, diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 4660d05b1e..bbbcf96538 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -103,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) { return { tables: connector.tables, error } } +async function buildFilteredSchema(datasource: Datasource, filter?: string[]) { + let { tables, error } = await buildSchemaHelper(datasource) + let finalTables = tables + if (filter) { + finalTables = {} + for (let key in tables) { + if ( + filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase()) + ) { + finalTables[key] = tables[key] + } + } + } + return { tables: finalTables, error } +} + export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() @@ -180,37 +196,22 @@ export async function information( export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() - const datasource = await sdk.datasources.get(ctx.params.datasourceId) const tablesFilter = ctx.request.body.tablesFilter + const datasource = await sdk.datasources.get(ctx.params.datasourceId) - let { tables, error } = await buildSchemaHelper(datasource) - if (tablesFilter) { - if (!datasource.entities) { - datasource.entities = {} - } - for (let key in tables) { - if ( - tablesFilter.some( - (filter: any) => filter.toLowerCase() === key.toLowerCase() - ) - ) { - datasource.entities[key] = tables[key] - } - } - } else { - datasource.entities = tables - } + const { tables, error } = await buildFilteredSchema(datasource, tablesFilter) + datasource.entities = tables setDefaultDisplayColumns(datasource) const dbResp = await db.put(datasource) datasource._rev = dbResp.rev const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource) - const response: any = { datasource: cleanedDatasource } + const res: any = { datasource: cleanedDatasource } if (error) { - response.error = error + res.error = error } - ctx.body = response + ctx.body = res } /** @@ -320,6 +321,7 @@ export async function save( const db = context.getAppDB() const plus = ctx.request.body.datasource.plus const fetchSchema = ctx.request.body.fetchSchema + const tablesFilter = ctx.request.body.tablesFilter const datasource = { _id: generateDatasourceID({ plus }), @@ -329,7 +331,10 @@ export async function save( let schemaError = null if (fetchSchema) { - const { tables, error } = await buildSchemaHelper(datasource) + const { tables, error } = await buildFilteredSchema( + datasource, + tablesFilter + ) schemaError = error datasource.entities = tables setDefaultDisplayColumns(datasource) diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index d692f17421..d0688a24d3 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -12,6 +12,7 @@ export interface UpdateDatasourceResponse { export interface CreateDatasourceRequest { datasource: Datasource fetchSchema?: boolean + tablesFilter: string[] } export interface VerifyDatasourceRequest { From ae1f7ae21a2815f78a10e5fde8a4815b3c53b268 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Jun 2023 18:50:50 +0100 Subject: [PATCH 014/127] Changing copy around datasource creation modals. --- .../modals/DatasourceConfigModal.svelte | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte index 3ab0b17840..acc04e4da3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte @@ -106,12 +106,16 @@ size="L" disabled={!isValid} > + + + {#if !fetchTableStep} + Connect your database to Budibase using the config below + {:else} + Choose what tables you want to sync with Budibase + {/if} + + {#if !fetchTableStep} - - Connect your database to Budibase using the config below. - - Date: Wed, 31 May 2023 13:00:33 +0200 Subject: [PATCH 015/127] Validate url --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 01fbc86700..86c32b80e0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 01fbc8670021c5a275c2a1a36ee18b984eeafad5 +Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 From c47656d423c81715175b824ecde95c11379407d6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 14:29:45 +0200 Subject: [PATCH 016/127] Save datasource --- packages/server/src/api/controllers/datasource.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index bbbcf96538..5f96536799 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -315,6 +315,12 @@ const preSaveAction: Partial> = { }, } +const preSaveAction: Partial> = { + [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { + await googleSetupCreationAuth(datasource.config as any) + }, +} + export async function save( ctx: UserCtx ) { From faf886f446811664603b6f425e2604dfd7fd6169 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 15:51:42 +0200 Subject: [PATCH 017/127] Change success message --- .../modals/GoogleDatasourceConfigModal.svelte | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 7b4808967d..40b625380d 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,7 +47,9 @@ [GoogleDatasouceConfigStep.SET_URL]: { confirmButtonText: "Connect", onConfirm: async () => { - if (integration.features[DatasourceFeature.CONNECTION_CHECKING]) { + const checkConnection = + integration.features[DatasourceFeature.CONNECTION_CHECKING] + if (checkConnection) { const resp = await validateDatasourceConfig(datasource) if (!resp.connected) { notifications.error(`Unable to connect - ${resp.error}`) @@ -58,7 +60,11 @@ try { const resp = await saveDatasource(datasource) $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource created successfully.`) + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) } catch (err) { notifications.error(err?.message ?? "Error saving datasource") // prevent the modal from closing From e9d4dd51bbc7a415ee973705c5cdc01bafa48f51 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 31 May 2023 16:00:55 +0200 Subject: [PATCH 018/127] Add sheets step --- .../modals/GoogleDatasourceConfigModal.svelte | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 40b625380d..09b17536f3 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -32,8 +32,9 @@ const integrationName = IntegrationNames[IntegrationTypes.GOOGLE_SHEETS] export const GoogleDatasouceConfigStep = { - AUTH: "Auth", - SET_URL: "Set_url", + AUTH: "auth", + SET_URL: "set_url", + SET_SHEETS: "set_sheets", } let step = continueSetupId @@ -43,8 +44,11 @@ let isValid = false const modalConfig = { - [GoogleDatasouceConfigStep.AUTH]: {}, + [GoogleDatasouceConfigStep.AUTH]: { + title: `Connect to ${integrationName}`, + }, [GoogleDatasouceConfigStep.SET_URL]: { + title: `Connect your spreadsheet`, confirmButtonText: "Connect", onConfirm: async () => { const checkConnection = @@ -57,14 +61,25 @@ } } + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) + + // prevent the modal from closing + return false + }, + }, + [GoogleDatasouceConfigStep.SET_SHEETS]: { + title: `Choose your sheets`, + confirmButtonText: "Fetch sheets", + onConfirm: async () => { try { const resp = await saveDatasource(datasource) $goto(`./datasource/${resp._id}`) - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + notifications.success(`Datasource created successfully.`) } catch (err) { notifications.error(err?.message ?? "Error saving datasource") // prevent the modal from closing @@ -76,7 +91,7 @@ {/if} + {#if step === GoogleDatasouceConfigStep.SET_SHEETS} + + Select which spreadsheet you want to connect. + + {/if} From 95c66edb657a08556cfbaa96e73eb8f6af6c0c44 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 11:48:11 +0100 Subject: [PATCH 019/127] Clean code --- packages/server/src/api/controllers/datasource.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 5f96536799..d765d96dcd 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -309,11 +309,6 @@ export async function update(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) } -const preSaveAction: Partial> = { - [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { - await googleSetupCreationAuth(datasource.config as any) - }, -} const preSaveAction: Partial> = { [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { From 258b3daf20411ca78f0d3a831df37209d2c8fb08 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 11:48:53 +0100 Subject: [PATCH 020/127] Update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 86c32b80e0..01fbc86700 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 86c32b80e08d2f19b57dcc2a3159667ac5a86c21 +Subproject commit 01fbc8670021c5a275c2a1a36ee18b984eeafad5 From 77bd2365573d18541c060657cee382337d1102f2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 12:13:56 +0100 Subject: [PATCH 021/127] Fetch info when enabled --- .../builder/src/builderStore/datasource.js | 6 +++ .../modals/GoogleDatasourceConfigModal.svelte | 52 ++++++++++++------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 49250d2628..427e03edbb 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -49,3 +49,9 @@ export async function getDatasourceInfo(config) { const datasource = prepareData(config) return await API.fetchInfoForDatasource(datasource) } + +export async function getDatasourceInfo(config) { + const datasource = prepareData(config) + const resp = await API.fetchTablesForDatasource(datasource) + return resp +} diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 09b17536f3..c74616fb3f 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -10,10 +10,14 @@ import GoogleButton from "../_components/GoogleButton.svelte" import { organisation } from "stores/portal" import { onMount } from "svelte" - import { validateDatasourceConfig } from "builderStore/datasource" + import { + validateDatasourceConfig, + getDatasourceInfo, + } from "builderStore/datasource" import cloneDeep from "lodash/cloneDeepWith" import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte" import { goto } from "@roxi/routify" + import { API } from "api" import { saveDatasource } from "builderStore/datasource" import { DatasourceFeature } from "@budibase/types" @@ -43,6 +47,18 @@ let isValid = false + const saveDatasourceAndRedirect = async () => { + try { + const resp = await saveDatasource(datasource) + $goto(`./datasource/${resp._id}`) + notifications.success(`Datasource created successfully.`) + } catch (err) { + notifications.error(err?.message ?? "Error saving datasource") + // prevent the modal from closing + return false + } + } + const modalConfig = { [GoogleDatasouceConfigStep.AUTH]: { title: `Connect to ${integrationName}`, @@ -61,12 +77,22 @@ } } - step = GoogleDatasouceConfigStep.SET_SHEETS - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + if (!integration.features[DatasourceFeature.FETCH_TABLE_NAMES]) { + saveDatasourceAndRedirect() + return + } + + getDatasourceInfo(datasource).then(i => { + debugger + console.log(i) + + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) + }) // prevent the modal from closing return false @@ -75,17 +101,7 @@ [GoogleDatasouceConfigStep.SET_SHEETS]: { title: `Choose your sheets`, confirmButtonText: "Fetch sheets", - onConfirm: async () => { - try { - const resp = await saveDatasource(datasource) - $goto(`./datasource/${resp._id}`) - notifications.success(`Datasource created successfully.`) - } catch (err) { - notifications.error(err?.message ?? "Error saving datasource") - // prevent the modal from closing - return false - } - }, + onConfirm: saveDatasourceAndRedirect, }, } From 5135ac36b7660dd5d2627de7490395bbb5a41c96 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 12:27:49 +0100 Subject: [PATCH 022/127] Fix fetching tables --- packages/server/src/integrations/googlesheets.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a792f49b57..a0c10d3655 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -150,7 +150,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { async testConnection(): Promise { try { - await setupCreationAuth(this.config) await this.connect() return { connected: true } } catch (e: any) { @@ -211,6 +210,8 @@ class GoogleSheetsIntegration implements DatasourcePlus { async connect() { try { + await setupCreationAuth(this.config) + // Initialise oAuth client let googleConfig = await configs.getGoogleDatasourceConfig() if (!googleConfig) { From 39848eef07b261e066acdb68a0fac020c725d024 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:52:44 +0100 Subject: [PATCH 023/127] Fetch sheets --- .../modals/GoogleDatasourceConfigModal.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index c74616fb3f..d01ae3de05 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,6 +47,8 @@ let isValid = false + let sheets + const saveDatasourceAndRedirect = async () => { try { const resp = await saveDatasource(datasource) @@ -82,9 +84,8 @@ return } - getDatasourceInfo(datasource).then(i => { - debugger - console.log(i) + const info = await getDatasourceInfo(datasource) + sheets = info.tableNames step = GoogleDatasouceConfigStep.SET_SHEETS notifications.success( @@ -92,7 +93,6 @@ ? "Connection Successful" : `Datasource created successfully.` ) - }) // prevent the modal from closing return false @@ -146,7 +146,7 @@ {/if} {#if step === GoogleDatasouceConfigStep.SET_SHEETS} - Select which spreadsheet you want to connect. + Select which spreadsheets you want to connect. {/if} From b128bdece5f24fb28f6e744a8f8850719210f18e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:55:30 +0100 Subject: [PATCH 024/127] Display sheets --- .../modals/GoogleDatasourceConfigModal.svelte | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index d01ae3de05..a98b76483d 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -47,7 +47,7 @@ let isValid = false - let sheets + let allSheets const saveDatasourceAndRedirect = async () => { try { @@ -85,14 +85,14 @@ } const info = await getDatasourceInfo(datasource) - sheets = info.tableNames + allSheets = info.tableNames - step = GoogleDatasouceConfigStep.SET_SHEETS - notifications.success( - checkConnection - ? "Connection Successful" - : `Datasource created successfully.` - ) + step = GoogleDatasouceConfigStep.SET_SHEETS + notifications.success( + checkConnection + ? "Connection Successful" + : `Datasource created successfully.` + ) // prevent the modal from closing return false @@ -101,7 +101,9 @@ [GoogleDatasouceConfigStep.SET_SHEETS]: { title: `Choose your sheets`, confirmButtonText: "Fetch sheets", - onConfirm: saveDatasourceAndRedirect, + onConfirm: async () => { + await saveDatasourceAndRedirect() + }, }, } @@ -147,6 +149,10 @@ {#if step === GoogleDatasouceConfigStep.SET_SHEETS} Select which spreadsheets you want to connect. + + {#each allSheets as sheet} + {sheet} + {/each} {/if} From f10d3d69733b136069744ea29351ccc2f841195f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 13:56:45 +0100 Subject: [PATCH 025/127] Update --- packages/builder/src/builderStore/datasource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 427e03edbb..2ed74f2e09 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -52,6 +52,6 @@ export async function getDatasourceInfo(config) { export async function getDatasourceInfo(config) { const datasource = prepareData(config) - const resp = await API.fetchTablesForDatasource(datasource) + const resp = await API.fetchInfoForDatasource(datasource) return resp } From bb901d94d2a14bcee35573bc46db03a9c2ec5d63 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 11:32:42 +0100 Subject: [PATCH 026/127] Clean code --- packages/builder/src/builderStore/datasource.js | 5 ----- packages/server/src/api/controllers/datasource.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/packages/builder/src/builderStore/datasource.js b/packages/builder/src/builderStore/datasource.js index 2ed74f2e09..6f9bd29fd8 100644 --- a/packages/builder/src/builderStore/datasource.js +++ b/packages/builder/src/builderStore/datasource.js @@ -45,11 +45,6 @@ export async function validateDatasourceConfig(config) { return await API.validateDatasource(datasource) } -export async function getDatasourceInfo(config) { - const datasource = prepareData(config) - return await API.fetchInfoForDatasource(datasource) -} - export async function getDatasourceInfo(config) { const datasource = prepareData(config) const resp = await API.fetchInfoForDatasource(datasource) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d765d96dcd..bbbcf96538 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -309,7 +309,6 @@ export async function update(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) } - const preSaveAction: Partial> = { [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { await googleSetupCreationAuth(datasource.config as any) From dd58aaf5cb3c1053652b2b46abf718efd7e97500 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 11:44:32 +0100 Subject: [PATCH 027/127] Use fancycheckboxgroup --- .../modals/GoogleDatasourceConfigModal.svelte | 11 ++++++----- packages/server/src/integrations/googlesheets.ts | 4 ---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index a98b76483d..93f8953cb6 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -5,6 +5,7 @@ Layout, Link, notifications, + FancyCheckboxGroup, } from "@budibase/bbui" import { IntegrationNames, IntegrationTypes } from "constants/backend" import GoogleButton from "../_components/GoogleButton.svelte" @@ -17,7 +18,6 @@ import cloneDeep from "lodash/cloneDeepWith" import IntegrationConfigForm from "../TableIntegrationMenu/IntegrationConfigForm.svelte" import { goto } from "@roxi/routify" - import { API } from "api" import { saveDatasource } from "builderStore/datasource" import { DatasourceFeature } from "@budibase/types" @@ -48,10 +48,13 @@ let isValid = false let allSheets + let selectedSheets const saveDatasourceAndRedirect = async () => { try { - const resp = await saveDatasource(datasource) + const resp = await saveDatasource(datasource, { + tablesFilter: selectedSheets, + }) $goto(`./datasource/${resp._id}`) notifications.success(`Datasource created successfully.`) } catch (err) { @@ -150,9 +153,7 @@ Select which spreadsheets you want to connect. - {#each allSheets as sheet} - {sheet} - {/each} + {/if} diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index a0c10d3655..3af564d059 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -274,10 +274,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { } async buildSchema(datasourceId: string, entities: Record) { - // not fully configured yet - if (!this.config.auth) { - return - } await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} From a14b29efac17adc5ce4efa7999a3f6294b45fb48 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 12:29:36 +0100 Subject: [PATCH 028/127] Improve fetch performance --- .../server/src/integrations/googlesheets.ts | 26 +++++++------ packages/shared-core/src/utils.ts | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 3af564d059..91bc310a44 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -21,7 +21,7 @@ import { buildExternalTableId, finaliseExternalTables } from "./utils" import { GoogleSpreadsheet, GoogleSpreadsheetRow } from "google-spreadsheet" import fetch from "node-fetch" import { cache, configs, context, HTTPError } from "@budibase/backend-core" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" import sdk from "../sdk" @@ -277,17 +277,21 @@ class GoogleSheetsIntegration implements DatasourcePlus { await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} - for (let sheet of sheets) { - // must fetch rows to determine schema - await sheet.getRows() + await utils.parallelForeach( + sheets, + async sheet => { + // must fetch rows to determine schema + await sheet.getRows({ limit: 0, offset: 0 }) - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - id - ) - } + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + id + ) + }, + 10 + ) const final = finaliseExternalTables(tables, entities) this.tables = final.tables this.schemaErrors = final.errors diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 720027d6a7..46a6585f89 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -4,3 +4,42 @@ export function unreachable( ) { throw new Error(message) } + +export async function parallelForeach( + items: T[], + task: (item: T) => Promise, + maxConcurrency: number +): Promise { + const promises: Promise[] = [] + let index = 0 + + const processItem = async (item: T) => { + try { + await task(item) + } finally { + processNext() + } + } + + const processNext = () => { + if (index >= items.length) { + // No more items to process + return + } + + const item = items[index] + index++ + + const promise = processItem(item) + promises.push(promise) + + if (promises.length >= maxConcurrency) { + Promise.race(promises).then(processNext) + } else { + processNext() + } + } + processNext() + + await Promise.all(promises) +} From d8fc37a76044691b142d0daeddf974e7ff6990f1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 14:08:47 +0100 Subject: [PATCH 029/127] Revert "Merge pull request #10808 from Budibase/undo_wrong_merge_from_develop" This reverts commit 4ccc98103c2e008d2f668101ca49d890c1a5da62, reversing changes made to 500424c651d0407767c37167615a1fd3d400d330. --- .../middleware/passport/datasource/google.ts | 35 ++- .../_components/GoogleButton.svelte | 14 +- .../modals/GoogleDatasourceConfigModal.svelte | 107 ++++++-- .../builder/app/[application]/data/new.svelte | 23 +- .../portal/settings/auth/google.svelte | 235 ++++++++++++++++++ .../builder/portal/settings/auth/index.svelte | 154 +----------- .../server/src/api/controllers/datasource.ts | 14 +- .../server/src/integrations/googlesheets.ts | 20 +- .../worker/src/api/controllers/global/auth.ts | 1 - 9 files changed, 391 insertions(+), 212 deletions(-) create mode 100644 packages/builder/src/pages/builder/portal/settings/auth/google.svelte diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 6fd4e9ff32..2f91e01d9a 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,10 +1,10 @@ import * as google from "../sso/google" import { Cookie } from "../../../constants" import { clearCookie, getCookie } from "../../../utils" -import { doWithDB } from "../../../db" import * as configs from "../../../configs" -import { BBContext, Database, SSOProfile } from "@budibase/types" +import { BBContext, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" +import { cache, utils } from "../../../" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy type Passport = { @@ -36,8 +36,8 @@ export async function preAuth( ssoSaveUserNoOp ) - if (!ctx.query.appId || !ctx.query.datasourceId) { - ctx.throw(400, "appId and datasourceId query params not present.") + if (!ctx.query.appId) { + ctx.throw(400, "appId query param not present.") } return passport.authenticate(strategy, { @@ -69,7 +69,7 @@ export async function postAuth( ( accessToken: string, refreshToken: string, - profile: SSOProfile, + _profile: SSOProfile, done: Function ) => { clearCookie(ctx, Cookie.DatasourceAuth) @@ -79,23 +79,16 @@ export async function postAuth( { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { const baseUrl = `/builder/app/${authStateCookie.appId}/data` - // update the DB for the datasource with all the user info - await doWithDB(authStateCookie.appId, async (db: Database) => { - let datasource - try { - datasource = await db.get(authStateCookie.datasourceId) - } catch (err: any) { - if (err.status === 404) { - ctx.redirect(baseUrl) - } + + const id = utils.newid() + await cache.store( + `datasource:creation:${authStateCookie.appId}:google:${id}`, + { + tokens, } - if (!datasource.config) { - datasource.config = {} - } - datasource.config.auth = { type: "google", ...tokens } - await db.put(datasource) - ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) - }) + ) + + ctx.redirect(`${baseUrl}/new?continue_google_setup=${id}`) } )(ctx, next) } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte index b7d70d88b7..ceb8fd7f4b 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -3,8 +3,6 @@ import { store } from "builderStore" import { auth } from "stores/portal" - export let preAuthStep - export let datasource export let disabled export let samePage @@ -15,18 +13,8 @@ class:disabled {disabled} on:click={async () => { - let ds = datasource let appId = $store.appId - if (!ds) { - const resp = await preAuthStep() - if (resp.datasource && resp.appId) { - ds = resp.datasource - appId = resp.appId - } else { - ds = resp - } - } - const url = `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${appId}` + const url = `/api/global/auth/${tenantId}/datasource/google?appId=${appId}` if (samePage) { window.location = url } else { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 0783a9fe53..7b4808967d 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -1,43 +1,110 @@ - - {#if isGoogleConfigured === true} - + {#if step === GoogleDatasouceConfigStep.AUTH} + + {#if isGoogleConfigured === true} + + Authenticate with your google account to use the {integrationName} integration. + + + {:else if isGoogleConfigured === false} Authenticate with your google account to use the {IntegrationNames[ - datasource.type - ]} integration.Google authentication is not enabled, please complete Google SSO + configuration. + Configure Google SSO + {/if} + {/if} + {#if step === GoogleDatasouceConfigStep.SET_URL} + + Add the URL of the sheet you want to connect. + + (isValid = e.detail)} + /> - save(datasource, true)} /> - {:else if isGoogleConfigured === false} - Google authentication is not enabled, please complete Google SSO - configuration. - Configure Google SSO {/if} diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte index fedaf013da..8ff974112b 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte @@ -17,6 +17,7 @@ import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte" import ICONS from "components/backend/DatasourceNavigator/icons/index.js" import FontAwesomeIcon from "components/common/FontAwesomeIcon.svelte" + import { onMount } from "svelte" let internalTableModal let externalDatasourceModal @@ -129,9 +130,19 @@ return integrationsArray } + let continueGoogleSetup + onMount(() => { + const urlParams = new URLSearchParams(window.location.search) + continueGoogleSetup = urlParams.get("continue_google_setup") + }) + const fetchIntegrations = async () => { const unsortedIntegrations = await API.getIntegrations() integrations = sortIntegrations(unsortedIntegrations) + + if (continueGoogleSetup) { + handleIntegrationSelect(IntegrationTypes.GOOGLE_SHEETS) + } } $: fetchIntegrations() @@ -141,9 +152,17 @@ - + { + continueGoogleSetup = null + }} +> {#if integration?.auth?.type === "google"} - + {:else} {/if} diff --git a/packages/builder/src/pages/builder/portal/settings/auth/google.svelte b/packages/builder/src/pages/builder/portal/settings/auth/google.svelte new file mode 100644 index 0000000000..82ab13cc8e --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/auth/google.svelte @@ -0,0 +1,235 @@ + + +{#if providers.google} + + + +
+ + Google +
+ + + To allow users to authenticate using their Google accounts, fill out the + fields below. Read the documentation for more information. + + + + {#each GoogleConfigFields.Google as field} +
+ +
+
+ +
+ {#if field.copyButton} +
copyToClipboard(field.placeholder)} + > + +
+ {/if} +
+
+ {/each} +
+ + +
+ +
+ +
+{/if} + + diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 38f5e0788b..36cf5c13a8 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -1,5 +1,4 @@
{/if} From ae7dbd4088d77767cd7ceb5184d2b860bfd679a6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 14:49:44 +0100 Subject: [PATCH 035/127] Rename text --- .../modals/GoogleDatasourceConfigModal.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 93f8953cb6..ef9ae8929b 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -153,7 +153,11 @@ Select which spreadsheets you want to connect. - + {/if} From cc52362fe371a2971ee4408226fce793d2a51d58 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Jun 2023 15:00:23 +0100 Subject: [PATCH 036/127] Use svelte class attribute --- packages/bbui/src/Form/Core/Checkbox.svelte | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/bbui/src/Form/Core/Checkbox.svelte b/packages/bbui/src/Form/Core/Checkbox.svelte index 36ab9d288e..3efc737bfb 100644 --- a/packages/bbui/src/Form/Core/Checkbox.svelte +++ b/packages/bbui/src/Form/Core/Checkbox.svelte @@ -20,11 +20,10 @@