From f4add0a180c220fa69163b21a3349357edbbcf6e Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 16 Dec 2021 10:03:59 +0000 Subject: [PATCH 01/35] Hide 'Body' tab for REST API datasources when using a GET query --- .../rest/[query]/index.svelte | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 39d02d5dd8..1c0a581b93 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -260,16 +260,18 @@ headings /> - - option.name} - getOptionValue={option => option.value} - /> - - + {#if query?.queryVerb !== "read"} + + option.name} + getOptionValue={option => option.value} + /> + + + {/if} {#if !$flags.queryTransformerBanner} From 49e813cc6cc41242404bbde54445baf41e57d9a6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 16 Dec 2021 10:11:49 +0000 Subject: [PATCH 02/35] Revert hiding body tab for GETs since already implemented in another pending PR --- .../rest/[query]/index.svelte | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 1c0a581b93..39d02d5dd8 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -260,18 +260,16 @@ headings /> - {#if query?.queryVerb !== "read"} - - option.name} - getOptionValue={option => option.value} - /> - - - {/if} + + option.name} + getOptionValue={option => option.value} + /> + + {#if !$flags.queryTransformerBanner} From af8d55ef323a7701dac46f2c490838f3006c8aaf Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 16 Dec 2021 11:56:14 +0000 Subject: [PATCH 03/35] Use fixed width for REST query verbs --- packages/builder/src/components/common/NavItem.svelte | 1 + packages/builder/src/helpers/data/utils.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 2d7dc745d1..d2074b69b1 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -133,5 +133,6 @@ .iconText { margin-top: 1px; font-size: var(--spectrum-global-dimension-font-size-50); + flex: 0 0 34px; } diff --git a/packages/builder/src/helpers/data/utils.js b/packages/builder/src/helpers/data/utils.js index 613f533043..1b8f1e04c5 100644 --- a/packages/builder/src/helpers/data/utils.js +++ b/packages/builder/src/helpers/data/utils.js @@ -84,7 +84,7 @@ export function customQueryIconText(datasource, query) { case "read": return "GET" case "delete": - return "DELETE" + return "DEL" case "patch": return "PATCH" } From 63ec10d73f717535b8113f6e16384f23893b9b03 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 08:22:04 +0000 Subject: [PATCH 04/35] Add new core implementations for fetching data and schema from datasources --- packages/client/src/api/datasources.js | 94 ----- packages/client/src/api/index.js | 1 - .../src/components/app/DataProvider.svelte | 318 +++------------ .../src/components/app/DataProvider2.svelte | 361 ++++++++++++++++++ .../components/app/blocks/CardsBlock.svelte | 4 +- .../components/app/blocks/TableBlock.svelte | 58 +-- .../src/components/app/forms/Form.svelte | 4 +- packages/client/src/sdk.js | 2 + packages/client/src/utils/fetch/QueryFetch.js | 42 ++ .../src/utils/fetch/RelationshipFetch.js | 23 ++ packages/client/src/utils/fetch/TableFetch.js | 324 ++++++++++++++++ packages/client/src/utils/fetch/ViewFetch.js | 34 ++ packages/client/src/utils/fetch/fetchData.js | 16 + packages/client/src/utils/schema.js | 49 +++ 14 files changed, 939 insertions(+), 391 deletions(-) delete mode 100644 packages/client/src/api/datasources.js create mode 100644 packages/client/src/components/app/DataProvider2.svelte create mode 100644 packages/client/src/utils/fetch/QueryFetch.js create mode 100644 packages/client/src/utils/fetch/RelationshipFetch.js create mode 100644 packages/client/src/utils/fetch/TableFetch.js create mode 100644 packages/client/src/utils/fetch/ViewFetch.js create mode 100644 packages/client/src/utils/fetch/fetchData.js create mode 100644 packages/client/src/utils/schema.js diff --git a/packages/client/src/api/datasources.js b/packages/client/src/api/datasources.js deleted file mode 100644 index 981d8301ca..0000000000 --- a/packages/client/src/api/datasources.js +++ /dev/null @@ -1,94 +0,0 @@ -import { cloneDeep } from "lodash/fp" -import { fetchTableData, fetchTableDefinition } from "./tables" -import { fetchViewData } from "./views" -import { fetchRelationshipData } from "./relationships" -import { FieldTypes } from "../constants" -import { executeQuery, fetchQueryDefinition } from "./queries" - -/** - * Fetches all rows for a particular Budibase data source. - */ -export const fetchDatasource = async dataSource => { - if (!dataSource || !dataSource.type) { - return [] - } - - // Fetch all rows in data source - const { type, tableId, fieldName } = dataSource - let rows = [] - if (type === "table") { - rows = await fetchTableData(tableId) - } else if (type === "view") { - rows = await fetchViewData(dataSource) - } else if (type === "query") { - // Set the default query params - let parameters = cloneDeep(dataSource.queryParams || {}) - for (let param of dataSource.parameters) { - if (!parameters[param.name]) { - parameters[param.name] = param.default - } - } - rows = await executeQuery({ queryId: dataSource._id, parameters }) - } else if (type === FieldTypes.LINK) { - rows = await fetchRelationshipData({ - rowId: dataSource.rowId, - tableId: dataSource.rowTableId, - fieldName, - }) - } - - // Enrich the result is always an array - return Array.isArray(rows) ? rows : [] -} - -/** - * Fetches the schema of any kind of datasource. - */ -export const fetchDatasourceSchema = async dataSource => { - if (!dataSource) { - return null - } - const { type } = dataSource - - // Nested providers should already have exposed their own schema - if (type === "provider") { - return dataSource.value?.schema - } - - // Field sources have their schema statically defined - if (type === "field") { - if (dataSource.fieldType === "attachment") { - return { - url: { - type: "string", - }, - name: { - type: "string", - }, - } - } else if (dataSource.fieldType === "array") { - return { - value: { - type: "string", - }, - } - } - } - - // Tables, views and links can be fetched by table ID - if ( - (type === "table" || type === "view" || type === "link") && - dataSource.tableId - ) { - const table = await fetchTableDefinition(dataSource.tableId) - return table?.schema - } - - // Queries can be fetched by query ID - if (type === "query" && dataSource._id) { - const definition = await fetchQueryDefinition(dataSource._id) - return definition?.schema - } - - return null -} diff --git a/packages/client/src/api/index.js b/packages/client/src/api/index.js index 1fd5b18139..d429eb437c 100644 --- a/packages/client/src/api/index.js +++ b/packages/client/src/api/index.js @@ -1,6 +1,5 @@ export * from "./rows" export * from "./auth" -export * from "./datasources" export * from "./tables" export * from "./attachments" export * from "./views" diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index e9d306cc3b..a33a9f14f2 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -8,6 +8,8 @@ luceneLimit, } from "builder/src/helpers/lucene" import Placeholder from "./Placeholder.svelte" + import { fetchData } from "utils/fetch/fetchData.js" + import { cloneDeep } from "lodash/fp" export let dataSource export let filter @@ -16,102 +18,62 @@ export let limit export let paginate - const { API, styleable, Provider, ActionTypes } = getContext("sdk") + const { styleable, Provider, ActionTypes } = getContext("sdk") const component = getContext("component") - // Loading flag every time data is being fetched - let loading = false - - // Loading flag for the initial load - // Mark as loaded if we have no datasource so we don't stall forever - let loaded = !dataSource - let schemaLoaded = false - - // Provider state - let rows = [] - let allRows = [] - let schema = {} - let bookmarks = [null] - let pageNumber = 0 - let query = null - let queryExtensions = {} - - // Sorting can be overridden at run time, so we can't use the prop directly - let currentSortColumn = sortColumn - let currentSortOrder = sortOrder - - // Reset the current sort state to props if props change - $: currentSortColumn = sortColumn - $: currentSortOrder = sortOrder - - $: defaultQuery = buildLuceneQuery(filter) - $: extendQuery(defaultQuery, queryExtensions) - $: internalTable = dataSource?.type === "table" - $: nestedProvider = dataSource?.type === "provider" - $: hasNextPage = bookmarks[pageNumber + 1] != null - $: hasPrevPage = pageNumber > 0 - $: getSchema(dataSource) - $: sortType = getSortType(schema, currentSortColumn) - - // Wait until schema loads before loading data, so that we can determine - // the correct sort type first time - $: { - if (schemaLoaded) { - fetchData( - dataSource, - schema, - query, - limit, - currentSortColumn, - currentSortOrder, - sortType, - paginate - ) - } + const createFetch = datasource => { + return fetchData(datasource, { + filter, + sortColumn, + sortOrder, + limit, + paginate, + }) } - // Reactively filter and sort rows if required + let fetch = fetchData() + + $: id = $component.id $: { - if (internalTable) { - // Internal tables are already processed server-side - rows = allRows - } else { - // For anything else we use client-side implementations to filter, sort - // and limit - const filtered = luceneQuery(allRows, query) - const sorted = luceneSort( - filtered, - currentSortColumn, - currentSortOrder, - sortType - ) - rows = luceneLimit(sorted, limit) - } + console.log("new datasource", id, dataSource) + fetch = createFetch(dataSource) } + $: fetch.update({ + filter, + sortColumn, + sortOrder, + limit, + paginate, + }) + // Build our action context $: actions = [ { type: ActionTypes.RefreshDatasource, - callback: () => refresh(), + callback: () => fetch.refresh(), metadata: { dataSource }, }, - { - type: ActionTypes.AddDataProviderQueryExtension, - callback: addQueryExtension, - }, - { - type: ActionTypes.RemoveDataProviderQueryExtension, - callback: removeQueryExtension, - }, + // { + // type: ActionTypes.AddDataProviderQueryExtension, + // callback: addQueryExtension, + // }, + // { + // type: ActionTypes.RemoveDataProviderQueryExtension, + // callback: removeQueryExtension, + // }, { type: ActionTypes.SetDataProviderSorting, callback: ({ column, order }) => { + let newOptions = {} if (column) { - currentSortColumn = column + newOptions.sortColumn = column } if (order) { - currentSortOrder = order + newOptions.sortOrder = order + } + if (Object.keys(newOptions)?.length) { + fetch.update(newOptions) } }, }, @@ -119,199 +81,25 @@ // Build our data context $: dataContext = { - rows, - schema, - rowsLength: rows.length, + rows: $fetch.rows, + schema: $fetch.schema, + rowsLength: $fetch.rows.length, // Undocumented properties. These aren't supposed to be used in builder // bindings, but are used internally by other components id: $component?.id, state: { - query, - sortColumn: currentSortColumn, - sortOrder: currentSortOrder, + query: $fetch.query, + sortColumn: $fetch.sortColumn, + sortOrder: $fetch.sortOrder, }, - loaded, - } - - const getSortType = (schema, sortColumn) => { - if (!schema || !sortColumn || !schema[sortColumn]) { - return "string" - } - const type = schema?.[sortColumn]?.type - return type === "number" ? "number" : "string" - } - - const refresh = async () => { - if (schemaLoaded && !nestedProvider) { - fetchData( - dataSource, - schema, - query, - limit, - currentSortColumn, - currentSortOrder, - sortType, - paginate - ) - } - } - - const fetchData = async ( - dataSource, - schema, - query, - limit, - sortColumn, - sortOrder, - sortType, - paginate - ) => { - loading = true - if (dataSource?.type === "table") { - // Sanity check sort column, as using a non-existant column will prevent - // results coming back at all - const sort = schema?.[sortColumn] ? sortColumn : undefined - - // For internal tables we use server-side processing - const res = await API.searchTable({ - tableId: dataSource.tableId, - query, - limit, - sort, - sortOrder: sortOrder?.toLowerCase() ?? "ascending", - sortType, - paginate, - }) - pageNumber = 0 - allRows = res.rows - if (res.hasNextPage) { - bookmarks = [null, res.bookmark] - } else { - bookmarks = [null] - } - } else if (dataSource?.type === "provider") { - // For providers referencing another provider, just use the rows it - // provides - allRows = dataSource?.value?.rows || [] - } else if (dataSource?.type === "field") { - // Field sources will be available from context. - // Enrich non object elements into object to ensure a valid schema. - const data = dataSource?.value || [] - if (Array.isArray(data) && data[0] && typeof data[0] !== "object") { - allRows = data.map(value => ({ value })) - } else { - allRows = data - } - } else { - // For other data sources like queries or views, fetch all rows from the - // server - allRows = await API.fetchDatasource(dataSource) - } - loading = false - loaded = true - } - - const getSchema = async dataSource => { - let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {} - - // Ensure there are "name" properties for all fields and that field schema - // are objects - Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => { - if (typeof fieldSchema === "string") { - newSchema[fieldName] = { - type: fieldSchema, - name: fieldName, - } - } else { - newSchema[fieldName] = { - ...fieldSchema, - name: fieldName, - } - } - }) - schema = newSchema - schemaLoaded = true - } - - const nextPage = async () => { - if (!hasNextPage || !internalTable) { - return - } - const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined - const res = await API.searchTable({ - tableId: dataSource?.tableId, - query, - bookmark: bookmarks[pageNumber + 1], - limit, - sort, - sortOrder: currentSortOrder?.toLowerCase() ?? "ascending", - sortType, - paginate: true, - }) - pageNumber++ - allRows = res.rows - if (res.hasNextPage) { - bookmarks[pageNumber + 1] = res.bookmark - } - } - - const prevPage = async () => { - if (!hasPrevPage || !internalTable) { - return - } - const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined - const res = await API.searchTable({ - tableId: dataSource?.tableId, - query, - bookmark: bookmarks[pageNumber - 1], - limit, - sort, - sortOrder: currentSortOrder?.toLowerCase() ?? "ascending", - sortType, - paginate: true, - }) - pageNumber-- - allRows = res.rows - } - - const addQueryExtension = (key, extension) => { - if (!key || !extension) { - return - } - queryExtensions = { ...queryExtensions, [key]: extension } - } - - const removeQueryExtension = key => { - if (!key) { - return - } - const newQueryExtensions = { ...queryExtensions } - delete newQueryExtensions[key] - queryExtensions = newQueryExtensions - } - - const extendQuery = (defaultQuery, extensions) => { - const extensionValues = Object.values(extensions || {}) - let extendedQuery = { ...defaultQuery } - extensionValues.forEach(extension => { - Object.entries(extension || {}).forEach(([operator, fields]) => { - extendedQuery[operator] = { - ...extendedQuery[operator], - ...fields, - } - }) - }) - - if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) { - query = extendedQuery - } + loaded: $fetch.loaded, }
- {#if !loaded} + {#if !$fetch.loaded}
@@ -321,14 +109,14 @@ {:else} {/if} - {#if paginate && internalTable} + {#if paginate && dataSource?.type === "table"} {/if} diff --git a/packages/client/src/components/app/DataProvider2.svelte b/packages/client/src/components/app/DataProvider2.svelte new file mode 100644 index 0000000000..b9f5287a49 --- /dev/null +++ b/packages/client/src/components/app/DataProvider2.svelte @@ -0,0 +1,361 @@ + + +
+ + {#if !loaded} +
+ +
+ {:else} + {#if $component.emptyState} + + {:else} + + {/if} + {#if paginate && internalTable} + + {/if} + {/if} + +
+ + diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte index ec631ede36..49c4ef5b8b 100644 --- a/packages/client/src/components/app/blocks/CardsBlock.svelte +++ b/packages/client/src/components/app/blocks/CardsBlock.svelte @@ -30,7 +30,7 @@ export let cardButtonOnClick export let linkColumn - const { API, styleable } = getContext("sdk") + const { fetchDatasourceSchema, styleable } = getContext("sdk") const context = getContext("context") const component = getContext("component") const schemaComponentMap = { @@ -111,7 +111,7 @@ // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await API.fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource) } } diff --git a/packages/client/src/components/app/blocks/TableBlock.svelte b/packages/client/src/components/app/blocks/TableBlock.svelte index 55937d1b7f..df99dee65c 100644 --- a/packages/client/src/components/app/blocks/TableBlock.svelte +++ b/packages/client/src/components/app/blocks/TableBlock.svelte @@ -26,7 +26,7 @@ export let titleButtonURL export let titleButtonPeek - const { API, styleable } = getContext("sdk") + const { fetchDatasourceSchema, styleable } = getContext("sdk") const context = getContext("context") const component = getContext("component") const schemaComponentMap = { @@ -40,6 +40,7 @@ let formId let dataProviderId let schema + let schemaLoaded = false $: fetchSchema(dataSource) $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema) @@ -89,8 +90,9 @@ // Load the datasource schema so we can determine column types const fetchSchema = async dataSource => { if (dataSource) { - schema = await API.fetchDatasourceSchema(dataSource) + schema = await fetchDatasourceSchema(dataSource) } + schemaLoaded = true } @@ -134,34 +136,36 @@
{/if} - + {#if schema} - + > + + + {/if} diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte index 1c6083aa5c..505c45bde0 100644 --- a/packages/client/src/components/app/forms/Form.svelte +++ b/packages/client/src/components/app/forms/Form.svelte @@ -10,7 +10,7 @@ export let actionType = "Create" const context = getContext("context") - const { API } = getContext("sdk") + const { API, fetchDatasourceSchema } = getContext("sdk") let loaded = false let schema @@ -61,7 +61,7 @@ // For all other cases, just grab the normal schema else { - const dataSourceSchema = await API.fetchDatasourceSchema(dataSource) + const dataSourceSchema = await fetchDatasourceSchema(dataSource) schema = dataSourceSchema || {} } diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js index ab1150658f..1c73361dc8 100644 --- a/packages/client/src/sdk.js +++ b/packages/client/src/sdk.js @@ -9,6 +9,7 @@ import { import { styleable } from "utils/styleable" import { linkable } from "utils/linkable" import { getAction } from "utils/getAction" +import { fetchDatasourceSchema } from "utils/schema.js" import Provider from "components/context/Provider.svelte" import { ActionTypes } from "constants" @@ -22,6 +23,7 @@ export default { styleable, linkable, getAction, + fetchDatasourceSchema, Provider, ActionTypes, } diff --git a/packages/client/src/utils/fetch/QueryFetch.js b/packages/client/src/utils/fetch/QueryFetch.js new file mode 100644 index 0000000000..80c031e7d9 --- /dev/null +++ b/packages/client/src/utils/fetch/QueryFetch.js @@ -0,0 +1,42 @@ +import TableFetch from "./TableFetch.js" +import { executeQuery, fetchQueryDefinition } from "api" +import { cloneDeep } from "lodash/fp.js" + +export default class ViewFetch extends TableFetch { + SupportsSearch = false + SupportsSort = false + SupportsPagination = false + + /** + * Fetches the schema for a view + * @param datasource the view datasource config + * @return {object} the view schema + */ + static async getSchema(datasource) { + if (!datasource?._id) { + return null + } + const definition = await fetchQueryDefinition(datasource._id) + return this.enrichSchema(definition?.schema) + } + + /** + * Fetches a single page of data from the remote resource + */ + async getData() { + const { datasource } = this.options + + // Set the default query params + let parameters = cloneDeep(datasource?.queryParams || {}) + for (let param of datasource?.parameters || {}) { + if (!parameters[param.name]) { + parameters[param.name] = param.default + } + } + + const res = await executeQuery({ queryId: datasource?._id, parameters }) + return { + rows: res || [], + } + } +} diff --git a/packages/client/src/utils/fetch/RelationshipFetch.js b/packages/client/src/utils/fetch/RelationshipFetch.js new file mode 100644 index 0000000000..d8396f7bfe --- /dev/null +++ b/packages/client/src/utils/fetch/RelationshipFetch.js @@ -0,0 +1,23 @@ +import TableFetch from "./TableFetch.js" +import { fetchRelationshipData } from "api" + +export default class ViewFetch extends TableFetch { + SupportsSearch = false + SupportsSort = false + SupportsPagination = false + + /** + * Fetches a single page of data from the remote resource + */ + async getData() { + const { datasource } = this.options + const res = await fetchRelationshipData({ + rowId: datasource?.rowId, + tableId: datasource?.rowTableId, + fieldName: datasource?.fieldName, + }) + return { + rows: res || [], + } + } +} diff --git a/packages/client/src/utils/fetch/TableFetch.js b/packages/client/src/utils/fetch/TableFetch.js new file mode 100644 index 0000000000..61fa8195a3 --- /dev/null +++ b/packages/client/src/utils/fetch/TableFetch.js @@ -0,0 +1,324 @@ +import { writable, derived, get } from "svelte/store" +import * as API from "api" +import { buildLuceneQuery } from "builder/src/helpers/lucene" +import { fetchTableDefinition } from "api" + +/** + * Parent class which handles the implementation of fetching data from an + * internal table or datasource plus. + * For other types of datasource, this class is overridden and extended. + */ +export default class TableFetch { + SupportsSearch = true + SupportsSort = true + SupportsPagination = true + + // Config + options = { + datasource: null, + schema: null, + limit: 10, + + // Search config + filter: null, + query: null, + + // Sorting config + sortColumn: null, + sortOrder: "ascending", + + // Pagination config + paginate: true, + } + + // State of the fetch + store = writable({ + rows: [], + schema: null, + loading: false, + loaded: false, + query: null, + pageNumber: 0, + cursor: null, + cursors: [], + }) + + /** + * Constructs a new DataFetch instance. + * @param opts the fetch options + */ + constructor(opts) { + this.options = { + ...this.options, + ...opts, + } + + // Bind all functions to properly scope "this" + this.getData = this.getData.bind(this) + this.getInitialData = this.getInitialData.bind(this) + this.refresh = this.refresh.bind(this) + this.update = this.update.bind(this) + this.hasNextPage = this.hasNextPage.bind(this) + this.hasPrevPage = this.hasPrevPage.bind(this) + this.nextPage = this.nextPage.bind(this) + this.prevPage = this.prevPage.bind(this) + + // Derive certain properties to return + this.derivedStore = derived(this.store, $store => { + return { + ...$store, + hasNextPage: this.hasNextPage($store), + hasPrevPage: this.hasPrevPage($store), + } + }) + + // Mark as loaded if we have no datasource + if (!this.options.datasource) { + this.store.update($store => ({ ...$store, loaded: true })) + return + } + + // Initially fetch data but don't bother waiting for the result + this.getInitialData() + } + + /** + * Extend the svelte store subscribe method to that instances of this class + * can be treated like stores + */ + get subscribe() { + return this.derivedStore.subscribe + } + + /** + * Fetches a fresh set of data from the server, resetting pagination + */ + async getInitialData() { + const { datasource, filter } = this.options + const tableId = datasource?.tableId + + // Ensure table ID exists + if (!tableId) { + return + } + + // Ensure schema exists and enrich it + let schema = this.options.schema + if (!schema) { + schema = await this.constructor.getSchema(datasource) + } + if (!schema) { + return + } + + // Build the lucene query + let query = this.options.query + if (!query) { + query = buildLuceneQuery(filter) + } + + // Update store + this.store.update($store => ({ ...$store, schema, query, loading: true })) + + // Actually fetch data + const page = await this.getData() + this.store.update($store => ({ + ...$store, + loading: false, + loaded: true, + pageNumber: 0, + rows: page.rows, + cursors: page.hasNextPage ? [null, page.cursor] : [null], + })) + } + + /** + * Fetches a single page of data from the remote resource + */ + async getData() { + const { datasource, limit, sortColumn, sortOrder, paginate } = this.options + const { tableId } = datasource + const { cursor, schema, query } = get(this.store) + + // Work out what sort type to use + const type = schema?.[sortColumn]?.type + const sortType = type === "number" ? "number" : "string" + + // Search table + const res = await API.searchTable({ + tableId, + query, + limit, + sort: sortColumn, + sortOrder: sortOrder?.toLowerCase() ?? "ascending", + sortType, + paginate, + bookmark: cursor, + }) + return { + rows: res?.rows || [], + hasNextPage: res?.hasNextPage || false, + cursor: res?.bookmark || null, + } + } + + /** + * Gets the schema definition for a table + * @param datasource the datasource definition + * @return {object} the schema + */ + static async getSchema(datasource) { + if (!datasource?.tableId) { + return null + } + const table = await fetchTableDefinition(datasource.tableId) + return this.enrichSchema(table?.schema) + } + + /** + * Enriches the schema and ensures that entries are objects with names + * @param schema the datasource schema + * @return {object} the enriched datasource schema + */ + static enrichSchema(schema) { + if (schema == null) { + return null + } + let enrichedSchema = {} + Object.entries(schema).forEach(([fieldName, fieldSchema]) => { + if (typeof fieldSchema === "string") { + enrichedSchema[fieldName] = { + type: fieldSchema, + name: fieldName, + } + } else { + enrichedSchema[fieldName] = { + ...fieldSchema, + name: fieldName, + } + } + }) + return enrichedSchema + } + + /** + * Resets the data set and updates options + * @param newOptions any new options + */ + async update(newOptions) { + // Check if any settings have actually changed + let refresh = false + const entries = Object.entries(newOptions || {}) + for (let [key, value] of entries) { + if (JSON.stringify(value) !== JSON.stringify(this.options[key])) { + refresh = true + break + } + } + if (!refresh) { + return + } + + // Assign new options and reload data + this.options = { + ...this.options, + ...newOptions, + } + await this.getInitialData() + } + + /** + * Loads the same page again + */ + async refresh() { + if (get(this.store).loading) { + return + } + const { rows } = await this.getData() + this.store.update($store => ({ ...$store, loading: true })) + this.store.update($store => ({ ...$store, rows, loading: false })) + } + + /** + * Determines whether there is a next page of data based on the state of the + * store + * @param state the current store state + * @return {boolean} whether there is a next page of data or not + */ + hasNextPage(state) { + return state.cursors[state.pageNumber + 1] != null + } + + /** + * Determines whether there is a previous page of data based on the state of + * the store + * @param state the current store state + * @return {boolean} whether there is a previous page of data or not + */ + hasPrevPage(state) { + return state.pageNumber > 0 + } + + /** + * Fetches the next page of data + */ + async nextPage() { + const state = get(this.derivedStore) + if (state.loading || !this.options.paginate || !state.hasNextPage) { + return + } + + // Fetch next page + const nextCursor = state.cursors[state.pageNumber + 1] + this.store.update($store => ({ + ...$store, + loading: true, + cursor: nextCursor, + })) + const { rows, hasNextPage, cursor } = await this.getData() + + // Update state + this.store.update($store => { + let { cursors, pageNumber } = $store + if (hasNextPage) { + cursors[pageNumber + 2] = cursor + } + return { + ...$store, + pageNumber: pageNumber + 1, + rows, + cursors, + loading: false, + } + }) + } + + /** + * Fetches the previous page of data + */ + async prevPage() { + const state = get(this.derivedStore) + if (state.loading || !this.options.paginate || !state.hasPrevPage) { + return + } + + // Fetch previous page + const prevCursor = state.cursors[state.pageNumber - 1] + this.store.update($store => ({ + ...$store, + loading: true, + cursor: prevCursor, + })) + const { rows } = await this.getData() + + // Update state + this.store.update($store => { + return { + ...$store, + pageNumber: $store.pageNumber - 1, + rows, + loading: false, + } + }) + } +} diff --git a/packages/client/src/utils/fetch/ViewFetch.js b/packages/client/src/utils/fetch/ViewFetch.js new file mode 100644 index 0000000000..beded054b5 --- /dev/null +++ b/packages/client/src/utils/fetch/ViewFetch.js @@ -0,0 +1,34 @@ +import TableFetch from "./TableFetch.js" +import { fetchTableDefinition, fetchViewData } from "api" + +export default class ViewFetch extends TableFetch { + SupportsSearch = false + SupportsSort = false + SupportsPagination = false + + /** + * Fetches the schema for a view + * @param datasource the view datasource config + * @return {object} the view schema + */ + static async getSchema(datasource) { + if (!datasource?.tableId) { + return null + } + const table = await fetchTableDefinition(datasource.tableId) + return this.enrichSchema(table?.views?.[datasource.name]?.schema) + } + + /** + * Fetches a single page of data from the remote resource + */ + async getData() { + const { datasource } = this.options + const res = await fetchViewData(datasource) + return { + rows: res || [], + } + } +} + +ViewFetch.getSchema() diff --git a/packages/client/src/utils/fetch/fetchData.js b/packages/client/src/utils/fetch/fetchData.js new file mode 100644 index 0000000000..bd14e849dc --- /dev/null +++ b/packages/client/src/utils/fetch/fetchData.js @@ -0,0 +1,16 @@ +import TableFetch from "./TableFetch.js" +import ViewFetch from "./ViewFetch.js" +import QueryFetch from "./QueryFetch.js" +import RelationshipFetch from "./RelationshipFetch.js" + +const DataFetchMap = { + table: TableFetch, + view: ViewFetch, + query: QueryFetch, + link: RelationshipFetch, +} + +export const fetchData = (datasource, options) => { + const Fetch = DataFetchMap[datasource?.type] || TableFetch + return new Fetch({ datasource, ...options }) +} diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js new file mode 100644 index 0000000000..022ce0a38d --- /dev/null +++ b/packages/client/src/utils/schema.js @@ -0,0 +1,49 @@ +import TableFetch from "./fetch/TableFetch.js" +import ViewFetch from "./fetch/ViewFetch.js" +import QueryFetch from "./fetch/QueryFetch.js" + +/** + * Fetches the schema of any kind of datasource. + */ +export const fetchDatasourceSchema = async datasource => { + const type = datasource?.type + + // Nested providers should already have exposed their own schema + if (type === "provider") { + return datasource.value?.schema + } + + // Field sources have their schema statically defined + if (type === "field") { + if (datasource.fieldType === "attachment") { + return { + url: { + type: "string", + }, + name: { + type: "string", + }, + } + } else if (datasource.fieldType === "array") { + return { + value: { + type: "string", + }, + } + } + } + + // All normal datasource schema can use their corresponsing implementations + // in the data fetch classes + if (type === "table" || type === "link") { + return TableFetch.getSchema(datasource) + } + if (type === "view") { + return ViewFetch.getSchema(datasource) + } + if (type === "query") { + return QueryFetch.getSchema(datasource) + } + + return null +} From 86065084c1c8689d096803243dfe8d1d8f9e3edd Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 08:22:40 +0000 Subject: [PATCH 05/35] Update core component handling to use svelte function to manually control props in order to avoid triggering additional reactive statements and improve performance --- .../client/src/components/Component.svelte | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index d8cdbd3f53..2ce97ee5fd 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -19,6 +19,9 @@ export let isScreen = false export let isBlock = false + // Ref to the svelte component + let ref + // Component settings are the un-enriched settings for this component that // need to be enriched at this level. // Nested settings are the un-enriched block settings that are to be passed on @@ -129,6 +132,11 @@ editing, }) + // Set initial props when the component mounts + $: if (ref?.$set) { + assignAllSettings() + } + // Extracts all settings from the component instance const getRawSettings = instance => { let validSettings = {} @@ -267,16 +275,30 @@ const cacheSettings = (enriched, nested, conditional) => { const allSettings = { ...enriched, ...nested, ...conditional } if (!cachedSettings) { - cachedSettings = allSettings + cachedSettings = { ...allSettings } } else { Object.keys(allSettings).forEach(key => { - if (!propsAreSame(allSettings[key], cachedSettings[key])) { + const same = propsAreSame(allSettings[key], cachedSettings[key]) + if (!same) { cachedSettings[key] = allSettings[key] + assignSetting(key, allSettings[key]) } }) } } + // Assigns the full set of settings to this component + const assignAllSettings = () => { + ref?.$set?.({ ...cachedSettings }) + } + + // Assigns a certain setting to this component. + // We manually use the svelte $set function to avoid triggering additional + // reactive statements. + const assignSetting = (key, value) => { + ref?.$$set?.({ [key]: value }) + } + // Generates a key used to determine when components need to fully remount. // Currently only toggling editing requires remounting. const getRenderKey = (id, editing) => { @@ -299,7 +321,7 @@ data-id={id} data-name={name} > - + {#if children.length} {#each children as child (child._id)} From 58e0daeb8c4fa89fbb045094017a9ba0f5c3be1b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 09:18:07 +0000 Subject: [PATCH 06/35] Remove initial render of components with undefined props by passing in initial props without triggering svelte invalidation --- .../client/src/components/Component.svelte | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 2ce97ee5fd..3e79c0bb15 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -3,7 +3,7 @@ - -
- - {#if title || enrichedSearchColumns?.length || showTitleButton} -
-
- {title || ""} +{#if schemaLoaded} + +
+ + {#if title || enrichedSearchColumns?.length || showTitleButton} +
+
+ {title || ""} +
+
+ {#if enrichedSearchColumns?.length} +
+ {#each enrichedSearchColumns as column} + + {/each} +
+ {/if} + {#if showTitleButton} + + {/if} +
-
- {#if enrichedSearchColumns?.length} -
- {#each enrichedSearchColumns as column} - - {/each} -
- {/if} - {#if showTitleButton} - - {/if} -
-
- {/if} - + {/if} + > + + - -
- +
+ +{/if} From 78a3f807a3f46bb34c39d3def8d67407f2ab8fd6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 10:43:27 +0000 Subject: [PATCH 12/35] Lint --- packages/client/src/components/Component.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte index 3e79c0bb15..d05f01f58a 100644 --- a/packages/client/src/components/Component.svelte +++ b/packages/client/src/components/Component.svelte @@ -3,7 +3,7 @@ @@ -270,6 +275,43 @@ /> + + + {#if !$flags.queryTransformerBanner} @@ -458,4 +500,9 @@ .auth-select { width: 200px; } + .pagination { + display: grid; + grid-template-columns: 1fr 1fr; + gap: var(--spacing-m); + } From f7759f6791aac1b066f983beb781221f63ffe7e5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 11:38:02 +0000 Subject: [PATCH 15/35] Allow saving pagination data in queries --- packages/server/src/api/controllers/query/validation.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/query/validation.js b/packages/server/src/api/controllers/query/validation.js index a17a752ca5..06850878cc 100644 --- a/packages/server/src/api/controllers/query/validation.js +++ b/packages/server/src/api/controllers/query/validation.js @@ -20,6 +20,7 @@ exports.queryValidation = () => { schema: Joi.object({}).required().unknown(true), transformer: Joi.string().optional(), flags: Joi.object().optional(), + pagination: Joi.object().optional(), }) } From 882067056cc45e3b9ce54c8ff7ab551a70a0d7e6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 18:39:22 +0000 Subject: [PATCH 16/35] Update query pagination to be stored in fields --- .../rest/[query]/index.svelte | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 37c5fce61e..0f78bfffcc 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -215,8 +215,8 @@ if (query && !query.fields.bodyType) { query.fields.bodyType = "none" } - if (query && !query.pagination) { - query.pagination = {} + if (query && !query.fields.pagination) { + query.fields.pagination = {} } dynamicVariables = restUtils.variablesToObject(datasource) }) @@ -296,34 +296,34 @@