From 441cc2fccf30d9acbdf703123b25b20bf0411326 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Fri, 17 Dec 2021 18:39:48 +0000 Subject: [PATCH] Update client lib data fetch models to support generic pagination --- packages/client/src/api/queries.js | 3 +- packages/client/src/utils/fetch/DataFetch.js | 83 ++++++++++++------- packages/client/src/utils/fetch/QueryFetch.js | 33 ++++++-- .../src/utils/fetch/RelationshipFetch.js | 2 +- packages/client/src/utils/fetch/TableFetch.js | 10 +-- packages/client/src/utils/fetch/ViewFetch.js | 13 ++- packages/client/src/utils/schema.js | 17 ++-- 7 files changed, 99 insertions(+), 62 deletions(-) diff --git a/packages/client/src/api/queries.js b/packages/client/src/api/queries.js index 834282fe73..e8972f657e 100644 --- a/packages/client/src/api/queries.js +++ b/packages/client/src/api/queries.js @@ -4,7 +4,7 @@ import API from "./api" /** * Executes a query against an external data connector. */ -export const executeQuery = async ({ queryId, parameters }) => { +export const executeQuery = async ({ queryId, pagination, parameters }) => { const query = await fetchQueryDefinition(queryId) if (query?.datasourceId == null) { notificationStore.actions.error("That query couldn't be found") @@ -14,6 +14,7 @@ export const executeQuery = async ({ queryId, parameters }) => { url: `/api/v2/queries/${queryId}`, body: { parameters, + pagination, }, }) if (res.error) { diff --git a/packages/client/src/utils/fetch/DataFetch.js b/packages/client/src/utils/fetch/DataFetch.js index e537e664d9..1238bd8fcd 100644 --- a/packages/client/src/utils/fetch/DataFetch.js +++ b/packages/client/src/utils/fetch/DataFetch.js @@ -12,7 +12,7 @@ import { fetchTableDefinition } from "api" * internal table or datasource plus. * For other types of datasource, this class is overridden and extended. */ -export default class TableFetch { +export default class DataFetch { // Feature flags supportsSearch = false supportsSort = false @@ -21,7 +21,6 @@ export default class TableFetch { // Config options = { datasource: null, - schema: null, limit: 10, // Search config @@ -53,23 +52,20 @@ export default class TableFetch { /** * Constructs a new DataFetch instance. * @param opts the fetch options - * @param flags the datasource feature flags */ - constructor(opts, flags) { + constructor(opts) { // Merge options with their default values this.options = { ...this.options, ...opts, } - // Update feature flags - this.supportsSearch = flags?.supportsSearch || false - this.supportsSort = flags?.supportsSort || false - this.supportsPagination = flags?.supportsPagination || false - // Bind all functions to properly scope "this" this.getData = this.getData.bind(this) + this.getPage = this.getPage.bind(this) this.getInitialData = this.getInitialData.bind(this) + this.determineFeatureFlags = this.determineFeatureFlags.bind(this) + this.enrichSchema = this.enrichSchema.bind(this) this.refresh = this.refresh.bind(this) this.update = this.update.bind(this) this.hasNextPage = this.hasNextPage.bind(this) @@ -116,11 +112,13 @@ export default class TableFetch { return } - // Ensure schema exists and enrich it - let schema = this.options.schema - if (!schema) { - schema = await this.constructor.getSchema(datasource) - } + // Fetch datasource definition and determine feature flags + const definition = await this.constructor.getDefinition(datasource) + this.determineFeatureFlags(definition) + + // Fetch and enrich schema + let schema = this.constructor.getSchema(datasource, definition) + schema = this.enrichSchema(schema) if (!schema) { return } @@ -142,13 +140,16 @@ export default class TableFetch { } // Update store - this.store.update($store => ({ ...$store, schema, query, loading: true })) + this.store.update($store => ({ + ...$store, + definition, + schema, + query, + loading: true, + })) // Actually fetch data const page = await this.getPage() - if (page.info) { - console.log("new info", page.info) - } this.store.update($store => ({ ...$store, loading: false, @@ -165,7 +166,7 @@ export default class TableFetch { */ async getPage() { const { sortColumn, sortOrder, sortType, limit } = this.options - const { query } = get(this.store) + const { query, pageNumber } = get(this.store) // Get the actual data let { rows, info, hasNextPage, cursor } = await this.getData() @@ -182,7 +183,8 @@ export default class TableFetch { // If we don't support pagination, do a client-side limit if (!this.supportsPagination) { - rows = luceneLimit(rows, limit) + rows = rows.slice(pageNumber * limit, (pageNumber + 1) * limit) + // rows = luceneLimit(rows, limit) } return { @@ -207,17 +209,27 @@ export default class TableFetch { } /** - * Gets the schema definition for a datasource. + * Gets the definition for this datasource. * Defaults to fetching a table definition. - * @param datasource the datasource definition - * @return {object} the schema + * @param datasource + * @return {object} the definition */ - static async getSchema(datasource) { + static async getDefinition(datasource) { if (!datasource?.tableId) { return null } - const table = await fetchTableDefinition(datasource.tableId) - return this.enrichSchema(table?.schema) + return await fetchTableDefinition(datasource.tableId) + } + + /** + * Gets the schema definition for a datasource. + * Defaults to getting the "schema" property of the definition. + * @param datasource the datasource + * @param definition the datasource definition + * @return {object} the schema + */ + static getSchema(datasource, definition) { + return definition?.schema } /** @@ -225,7 +237,7 @@ export default class TableFetch { * @param schema the datasource schema * @return {object} the enriched datasource schema */ - static enrichSchema(schema) { + enrichSchema(schema) { if (schema == null) { return null } @@ -246,6 +258,17 @@ export default class TableFetch { return enrichedSchema } + /** + * Determine the feature flag for this datasource definition + * @param definition + */ + // eslint-disable-next-line no-unused-vars + determineFeatureFlags(definition) { + this.supportsSearch = false + this.supportsSort = false + this.supportsPagination = false + } + /** * Resets the data set and updates options * @param newOptions any new options @@ -319,6 +342,7 @@ export default class TableFetch { ...$store, loading: true, cursor: nextCursor, + pageNumber: $store.pageNumber + 1, })) const { rows, info, hasNextPage, cursor } = await this.getPage() @@ -326,11 +350,10 @@ export default class TableFetch { this.store.update($store => { let { cursors, pageNumber } = $store if (hasNextPage) { - cursors[pageNumber + 2] = cursor + cursors[pageNumber + 1] = cursor } return { ...$store, - pageNumber: pageNumber + 1, rows, info, cursors, @@ -354,6 +377,7 @@ export default class TableFetch { ...$store, loading: true, cursor: prevCursor, + pageNumber: $store.pageNumber - 1, })) const { rows, info } = await this.getPage() @@ -361,7 +385,6 @@ export default class TableFetch { this.store.update($store => { return { ...$store, - pageNumber: $store.pageNumber - 1, rows, info, loading: false, diff --git a/packages/client/src/utils/fetch/QueryFetch.js b/packages/client/src/utils/fetch/QueryFetch.js index d16189fe76..097fff1250 100644 --- a/packages/client/src/utils/fetch/QueryFetch.js +++ b/packages/client/src/utils/fetch/QueryFetch.js @@ -1,18 +1,25 @@ import DataFetch from "./DataFetch.js" import { executeQuery, fetchQueryDefinition } from "api" import { cloneDeep } from "lodash/fp" +import { get } from "svelte/store" -export default class ViewFetch extends DataFetch { - static async getSchema(datasource) { +export default class QueryFetch extends DataFetch { + determineFeatureFlags(definition) { + console.log("pagination config", definition?.pagination) + this.supportsPagination = + definition?.fields?.pagination?.type != null && + definition?.fields?.pagination?.pageParam != null + } + + static async getDefinition(datasource) { if (!datasource?._id) { return null } - const definition = await fetchQueryDefinition(datasource._id) - return this.enrichSchema(definition?.schema) + return await fetchQueryDefinition(datasource._id) } async getData() { - const { datasource } = this.options + const { datasource, limit } = this.options // Set the default query params let parameters = cloneDeep(datasource?.queryParams || {}) @@ -22,13 +29,21 @@ export default class ViewFetch extends DataFetch { } } - const { data, ...rest } = await executeQuery({ - queryId: datasource?._id, - parameters, - }) + // Add pagination to query if supported + let queryPayload = { queryId: datasource?._id, parameters } + if (this.supportsPagination) { + const { cursor, definition, pageNumber } = get(this.store) + const { type } = definition.fields.pagination.type + const page = type === "page" ? pageNumber : cursor + queryPayload.pagination = { page, limit } + } + + const { data, pagination, ...rest } = await executeQuery(queryPayload) return { rows: data || [], info: rest, + cursor: pagination?.page, + hasNextPage: data?.length === limit && limit > 0, } } } diff --git a/packages/client/src/utils/fetch/RelationshipFetch.js b/packages/client/src/utils/fetch/RelationshipFetch.js index 27178901da..b8adf686f1 100644 --- a/packages/client/src/utils/fetch/RelationshipFetch.js +++ b/packages/client/src/utils/fetch/RelationshipFetch.js @@ -1,7 +1,7 @@ import DataFetch from "./DataFetch.js" import { fetchRelationshipData } from "api" -export default class ViewFetch extends DataFetch { +export default class RelationshipFetch extends DataFetch { async getData() { const { datasource } = this.options const res = await fetchRelationshipData({ diff --git a/packages/client/src/utils/fetch/TableFetch.js b/packages/client/src/utils/fetch/TableFetch.js index 5217ec5152..bb6d913321 100644 --- a/packages/client/src/utils/fetch/TableFetch.js +++ b/packages/client/src/utils/fetch/TableFetch.js @@ -3,12 +3,10 @@ import DataFetch from "./DataFetch.js" import { searchTable } from "api" export default class TableFetch extends DataFetch { - constructor(opts) { - super(opts, { - supportsSearch: true, - supportsSort: true, - supportsPagination: true, - }) + determineFeatureFlags() { + this.supportsSearch = true + this.supportsSort = true + this.supportsPagination = true } async getData() { diff --git a/packages/client/src/utils/fetch/ViewFetch.js b/packages/client/src/utils/fetch/ViewFetch.js index 157d3f4201..639cbf9450 100644 --- a/packages/client/src/utils/fetch/ViewFetch.js +++ b/packages/client/src/utils/fetch/ViewFetch.js @@ -1,18 +1,17 @@ import DataFetch from "./DataFetch.js" -import { fetchTableDefinition, fetchViewData } from "api" +import { fetchViewData } from "api" export default class ViewFetch extends DataFetch { - static async getSchema(datasource) { - if (!datasource?.tableId) { - return null - } - const table = await fetchTableDefinition(datasource.tableId) - return this.enrichSchema(table?.views?.[datasource.name]?.schema) + static async getSchema(datasource, definition) { + const schema = definition?.views?.[datasource.name]?.schema + console.log(schema) + return schema } async getData() { const { datasource } = this.options const res = await fetchViewData(datasource) + console.log(res) return { rows: res || [], } diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js index 022ce0a38d..bea1f217b8 100644 --- a/packages/client/src/utils/schema.js +++ b/packages/client/src/utils/schema.js @@ -35,14 +35,15 @@ export const fetchDatasourceSchema = async datasource => { // All normal datasource schema can use their corresponsing implementations // in the data fetch classes - if (type === "table" || type === "link") { - return TableFetch.getSchema(datasource) - } - if (type === "view") { - return ViewFetch.getSchema(datasource) - } - if (type === "query") { - return QueryFetch.getSchema(datasource) + const handler = { + table: TableFetch, + link: TableFetch, + view: ViewFetch, + query: QueryFetch, + }[type] + if (handler) { + const definition = await handler.getDefinition(datasource) + return handler.getSchema(datasource, definition) } return null