MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add new core implementations for fetching data and schema from datasources

This commit is contained in:
Andrew Kingston 2021-12-17 08:22:04 +00:00
parent 7bf0a50fba
commit 4bf200fcc5
14 changed files with 939 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
packages/client/src/api/datasources.js
View File

@ -1,94 +0,0 @@
import { cloneDeep } from "lodash/fp"
import { fetchTableData, fetchTableDefinition } from "./tables"
import { fetchViewData } from "./views"
import { fetchRelationshipData } from "./relationships"
import { FieldTypes } from "../constants"
import { executeQuery, fetchQueryDefinition } from "./queries"
/**
* Fetches all rows for a particular Budibase data source.
*/
export const fetchDatasource = async dataSource => {
if (!dataSource || !dataSource.type) {
return []
}
// Fetch all rows in data source
const { type, tableId, fieldName } = dataSource
let rows = []
if (type === "table") {
rows = await fetchTableData(tableId)
} else if (type === "view") {
rows = await fetchViewData(dataSource)
} else if (type === "query") {
// Set the default query params
let parameters = cloneDeep(dataSource.queryParams || {})
for (let param of dataSource.parameters) {
if (!parameters[param.name]) {
parameters[param.name] = param.default
}
}
rows = await executeQuery({ queryId: dataSource._id, parameters })
} else if (type === FieldTypes.LINK) {
rows = await fetchRelationshipData({
rowId: dataSource.rowId,
tableId: dataSource.rowTableId,
fieldName,
})
}
// Enrich the result is always an array
return Array.isArray(rows) ? rows : []
}
/**
* Fetches the schema of any kind of datasource.
*/
export const fetchDatasourceSchema = async dataSource => {
if (!dataSource) {
return null
}
const { type } = dataSource
// Nested providers should already have exposed their own schema
if (type === "provider") {
return dataSource.value?.schema
}
// Field sources have their schema statically defined
if (type === "field") {
if (dataSource.fieldType === "attachment") {
return {
url: {
type: "string",
},
name: {
type: "string",
},
}
} else if (dataSource.fieldType === "array") {
return {
value: {
type: "string",
},
}
}
}
// Tables, views and links can be fetched by table ID
if (
(type === "table" || type === "view" || type === "link") &&
dataSource.tableId
) {
const table = await fetchTableDefinition(dataSource.tableId)
return table?.schema
}
// Queries can be fetched by query ID
if (type === "query" && dataSource._id) {
const definition = await fetchQueryDefinition(dataSource._id)
return definition?.schema
}
return null
}

1
packages/client/src/api/index.js
View File

@ -1,6 +1,5 @@
export * from "./rows"
export * from "./auth"
export * from "./datasources"
export * from "./tables"
export * from "./attachments"
export * from "./views"

316
packages/client/src/components/app/DataProvider.svelte
View File

@ -8,6 +8,8 @@
luceneLimit,
} from "builder/src/helpers/lucene"
import Placeholder from "./Placeholder.svelte"
import { fetchData } from "utils/fetch/fetchData.js"
import { cloneDeep } from "lodash/fp"
export let dataSource
export let filter
@ -16,102 +18,62 @@
export let limit
export let paginate
const { API, styleable, Provider, ActionTypes } = getContext("sdk")
const { styleable, Provider, ActionTypes } = getContext("sdk")
const component = getContext("component")
// Loading flag every time data is being fetched
let loading = false
// Loading flag for the initial load
// Mark as loaded if we have no datasource so we don't stall forever
let loaded = !dataSource
let schemaLoaded = false
// Provider state
let rows = []
let allRows = []
let schema = {}
let bookmarks = [null]
let pageNumber = 0
let query = null
let queryExtensions = {}
// Sorting can be overridden at run time, so we can't use the prop directly
let currentSortColumn = sortColumn
let currentSortOrder = sortOrder
// Reset the current sort state to props if props change
$: currentSortColumn = sortColumn
$: currentSortOrder = sortOrder
$: defaultQuery = buildLuceneQuery(filter)
$: extendQuery(defaultQuery, queryExtensions)
$: internalTable = dataSource?.type === "table"
$: nestedProvider = dataSource?.type === "provider"
$: hasNextPage = bookmarks[pageNumber + 1] != null
$: hasPrevPage = pageNumber > 0
$: getSchema(dataSource)
$: sortType = getSortType(schema, currentSortColumn)
// Wait until schema loads before loading data, so that we can determine
// the correct sort type first time
$: {
if (schemaLoaded) {
fetchData(
dataSource,
schema,
query,
const createFetch = datasource => {
return fetchData(datasource, {
filter,
sortColumn,
sortOrder,
limit,
currentSortColumn,
currentSortOrder,
sortType,
paginate
)
}
paginate,
})
}
// Reactively filter and sort rows if required
let fetch = fetchData()
$: id = $component.id
$: {
if (internalTable) {
// Internal tables are already processed server-side
rows = allRows
} else {
// For anything else we use client-side implementations to filter, sort
// and limit
const filtered = luceneQuery(allRows, query)
const sorted = luceneSort(
filtered,
currentSortColumn,
currentSortOrder,
sortType
)
rows = luceneLimit(sorted, limit)
}
console.log("new datasource", id, dataSource)
fetch = createFetch(dataSource)
}
$: fetch.update({
filter,
sortColumn,
sortOrder,
limit,
paginate,
})
// Build our action context
$: actions = [
{
type: ActionTypes.RefreshDatasource,
callback: () => refresh(),
callback: () => fetch.refresh(),
metadata: { dataSource },
},
{
type: ActionTypes.AddDataProviderQueryExtension,
callback: addQueryExtension,
},
{
type: ActionTypes.RemoveDataProviderQueryExtension,
callback: removeQueryExtension,
},
// {
// type: ActionTypes.AddDataProviderQueryExtension,
// callback: addQueryExtension,
// },
// {
// type: ActionTypes.RemoveDataProviderQueryExtension,
// callback: removeQueryExtension,
// },
{
type: ActionTypes.SetDataProviderSorting,
callback: ({ column, order }) => {
let newOptions = {}
if (column) {
currentSortColumn = column
newOptions.sortColumn = column
}
if (order) {
currentSortOrder = order
newOptions.sortOrder = order
}
if (Object.keys(newOptions)?.length) {
fetch.update(newOptions)
}
},
},
@ -119,199 +81,25 @@
// Build our data context
$: dataContext = {
rows,
schema,
rowsLength: rows.length,
rows: $fetch.rows,
schema: $fetch.schema,
rowsLength: $fetch.rows.length,
// Undocumented properties. These aren't supposed to be used in builder
// bindings, but are used internally by other components
id: $component?.id,
state: {
query,
sortColumn: currentSortColumn,
sortOrder: currentSortOrder,
query: $fetch.query,
sortColumn: $fetch.sortColumn,
sortOrder: $fetch.sortOrder,
},
loaded,
}
const getSortType = (schema, sortColumn) => {
if (!schema || !sortColumn || !schema[sortColumn]) {
return "string"
}
const type = schema?.[sortColumn]?.type
return type === "number" ? "number" : "string"
}
const refresh = async () => {
if (schemaLoaded && !nestedProvider) {
fetchData(
dataSource,
schema,
query,
limit,
currentSortColumn,
currentSortOrder,
sortType,
paginate
)
}
}
const fetchData = async (
dataSource,
schema,
query,
limit,
sortColumn,
sortOrder,
sortType,
paginate
) => {
loading = true
if (dataSource?.type === "table") {
// Sanity check sort column, as using a non-existant column will prevent
// results coming back at all
const sort = schema?.[sortColumn] ? sortColumn : undefined
// For internal tables we use server-side processing
const res = await API.searchTable({
tableId: dataSource.tableId,
query,
limit,
sort,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate,
})
pageNumber = 0
allRows = res.rows
if (res.hasNextPage) {
bookmarks = [null, res.bookmark]
} else {
bookmarks = [null]
}
} else if (dataSource?.type === "provider") {
// For providers referencing another provider, just use the rows it
// provides
allRows = dataSource?.value?.rows || []
} else if (dataSource?.type === "field") {
// Field sources will be available from context.
// Enrich non object elements into object to ensure a valid schema.
const data = dataSource?.value || []
if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
allRows = data.map(value => ({ value }))
} else {
allRows = data
}
} else {
// For other data sources like queries or views, fetch all rows from the
// server
allRows = await API.fetchDatasource(dataSource)
}
loading = false
loaded = true
}
const getSchema = async dataSource => {
let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {}
// Ensure there are "name" properties for all fields and that field schema
// are objects
Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => {
if (typeof fieldSchema === "string") {
newSchema[fieldName] = {
type: fieldSchema,
name: fieldName,
}
} else {
newSchema[fieldName] = {
...fieldSchema,
name: fieldName,
}
}
})
schema = newSchema
schemaLoaded = true
}
const nextPage = async () => {
if (!hasNextPage || !internalTable) {
return
}
const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
const res = await API.searchTable({
tableId: dataSource?.tableId,
query,
bookmark: bookmarks[pageNumber + 1],
limit,
sort,
sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber++
allRows = res.rows
if (res.hasNextPage) {
bookmarks[pageNumber + 1] = res.bookmark
}
}
const prevPage = async () => {
if (!hasPrevPage || !internalTable) {
return
}
const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
const res = await API.searchTable({
tableId: dataSource?.tableId,
query,
bookmark: bookmarks[pageNumber - 1],
limit,
sort,
sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber--
allRows = res.rows
}
const addQueryExtension = (key, extension) => {
if (!key || !extension) {
return
}
queryExtensions = { ...queryExtensions, [key]: extension }
}
const removeQueryExtension = key => {
if (!key) {
return
}
const newQueryExtensions = { ...queryExtensions }
delete newQueryExtensions[key]
queryExtensions = newQueryExtensions
}
const extendQuery = (defaultQuery, extensions) => {
const extensionValues = Object.values(extensions || {})
let extendedQuery = { ...defaultQuery }
extensionValues.forEach(extension => {
Object.entries(extension || {}).forEach(([operator, fields]) => {
extendedQuery[operator] = {
...extendedQuery[operator],
...fields,
}
})
})
if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) {
query = extendedQuery
}
loaded: $fetch.loaded,
}
</script>
<div use:styleable={$component.styles} class="container">
<Provider {actions} data={dataContext}>
{#if !loaded}
{#if !$fetch.loaded}
<div class="loading">
<ProgressCircle />
</div>
@ -321,14 +109,14 @@
{:else}
<slot />
{/if}
{#if paginate && internalTable}
{#if paginate && dataSource?.type === "table"}
<div class="pagination">
<Pagination
page={pageNumber + 1}
{hasPrevPage}
{hasNextPage}
goToPrevPage={prevPage}
goToNextPage={nextPage}
page={$fetch.pageNumber + 1}
hasPrevPage={$fetch.hasPrevPage}
hasNextPage={$fetch.hasNextPage}
goToPrevPage={fetch.prevPage}
goToNextPage={fetch.nextPage}
/>
</div>
{/if}

361
packages/client/src/components/app/DataProvider2.svelte Normal file
View File

@ -0,0 +1,361 @@
<script>
import { getContext } from "svelte"
import { ProgressCircle, Pagination } from "@budibase/bbui"
import {
buildLuceneQuery,
luceneQuery,
luceneSort,
luceneLimit,
} from "builder/src/helpers/lucene"
import Placeholder from "./Placeholder.svelte"
import { fetchData } from "utils/fetch/fetchData.js"
export let dataSource
export let filter
export let sortColumn
export let sortOrder
export let limit
export let paginate
const { API, styleable, Provider, ActionTypes } = getContext("sdk")
const component = getContext("component")
// Loading flag every time data is being fetched
let loading = false
// Loading flag for the initial load
// Mark as loaded if we have no datasource so we don't stall forever
let loaded = !dataSource
let schemaLoaded = false
// Provider state
let rows = []
let allRows = []
let schema = {}
let bookmarks = [null]
let pageNumber = 0
let query = null
let queryExtensions = {}
// Sorting can be overridden at run time, so we can't use the prop directly
let currentSortColumn = sortColumn
let currentSortOrder = sortOrder
// Reset the current sort state to props if props change
$: currentSortColumn = sortColumn
$: currentSortOrder = sortOrder
$: defaultQuery = buildLuceneQuery(filter)
$: extendQuery(defaultQuery, queryExtensions)
$: internalTable = dataSource?.type === "table"
$: nestedProvider = dataSource?.type === "provider"
$: hasNextPage = bookmarks[pageNumber + 1] != null
$: hasPrevPage = pageNumber > 0
$: getSchema(dataSource)
$: sortType = getSortType(schema, currentSortColumn)
// Wait until schema loads before loading data, so that we can determine
// the correct sort type first time
$: {
if (schemaLoaded) {
fetchData(
dataSource,
schema,
query,
limit,
currentSortColumn,
currentSortOrder,
sortType,
paginate
)
}
}
// Reactively filter and sort rows if required
$: {
if (internalTable) {
// Internal tables are already processed server-side
rows = allRows
} else {
// For anything else we use client-side implementations to filter, sort
// and limit
const filtered = luceneQuery(allRows, query)
const sorted = luceneSort(
filtered,
currentSortColumn,
currentSortOrder,
sortType
)
rows = luceneLimit(sorted, limit)
}
}
// Build our action context
$: actions = [
{
type: ActionTypes.RefreshDatasource,
callback: () => refresh(),
metadata: { dataSource },
},
{
type: ActionTypes.AddDataProviderQueryExtension,
callback: addQueryExtension,
},
{
type: ActionTypes.RemoveDataProviderQueryExtension,
callback: removeQueryExtension,
},
{
type: ActionTypes.SetDataProviderSorting,
callback: ({ column, order }) => {
if (column) {
currentSortColumn = column
}
if (order) {
currentSortOrder = order
}
},
},
]
// Build our data context
$: dataContext = {
rows,
schema,
rowsLength: rows.length,
// Undocumented properties. These aren't supposed to be used in builder
// bindings, but are used internally by other components
id: $component?.id,
state: {
query,
sortColumn: currentSortColumn,
sortOrder: currentSortOrder,
},
loaded,
}
const getSortType = (schema, sortColumn) => {
if (!schema || !sortColumn || !schema[sortColumn]) {
return "string"
}
const type = schema?.[sortColumn]?.type
return type === "number" ? "number" : "string"
}
const refresh = async () => {
if (schemaLoaded && !nestedProvider) {
fetchData(
dataSource,
schema,
query,
limit,
currentSortColumn,
currentSortOrder,
sortType,
paginate
)
}
}
const fetchData = async (
dataSource,
schema,
query,
limit,
sortColumn,
sortOrder,
sortType,
paginate
) => {
loading = true
if (dataSource?.type === "table") {
// Sanity check sort column, as using a non-existant column will prevent
// results coming back at all
const sort = schema?.[sortColumn] ? sortColumn : undefined
// For internal tables we use server-side processing
const res = await API.searchTable({
tableId: dataSource.tableId,
query,
limit,
sort,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate,
})
pageNumber = 0
allRows = res.rows
if (res.hasNextPage) {
bookmarks = [null, res.bookmark]
} else {
bookmarks = [null]
}
} else if (dataSource?.type === "provider") {
// For providers referencing another provider, just use the rows it
// provides
allRows = dataSource?.value?.rows || []
} else if (dataSource?.type === "field") {
// Field sources will be available from context.
// Enrich non object elements into object to ensure a valid schema.
const data = dataSource?.value || []
if (Array.isArray(data) && data[0] && typeof data[0] !== "object") {
allRows = data.map(value => ({ value }))
} else {
allRows = data
}
} else {
// For other data sources like queries or views, fetch all rows from the
// server
allRows = await API.fetchDatasource(dataSource)
}
loading = false
loaded = true
}
const getSchema = async dataSource => {
let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {}
// Ensure there are "name" properties for all fields and that field schema
// are objects
Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => {
if (typeof fieldSchema === "string") {
newSchema[fieldName] = {
type: fieldSchema,
name: fieldName,
}
} else {
newSchema[fieldName] = {
...fieldSchema,
name: fieldName,
}
}
})
schema = newSchema
schemaLoaded = true
}
const nextPage = async () => {
if (!hasNextPage || !internalTable) {
return
}
const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
const res = await API.searchTable({
tableId: dataSource?.tableId,
query,
bookmark: bookmarks[pageNumber + 1],
limit,
sort,
sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber++
allRows = res.rows
if (res.hasNextPage) {
bookmarks[pageNumber + 1] = res.bookmark
}
}
const prevPage = async () => {
if (!hasPrevPage || !internalTable) {
return
}
const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
const res = await API.searchTable({
tableId: dataSource?.tableId,
query,
bookmark: bookmarks[pageNumber - 1],
limit,
sort,
sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate: true,
})
pageNumber--
allRows = res.rows
}
const addQueryExtension = (key, extension) => {
if (!key || !extension) {
return
}
queryExtensions = { ...queryExtensions, [key]: extension }
}
const removeQueryExtension = key => {
if (!key) {
return
}
const newQueryExtensions = { ...queryExtensions }
delete newQueryExtensions[key]
queryExtensions = newQueryExtensions
}
const extendQuery = (defaultQuery, extensions) => {
const extensionValues = Object.values(extensions || {})
let extendedQuery = { ...defaultQuery }
extensionValues.forEach(extension => {
Object.entries(extension || {}).forEach(([operator, fields]) => {
extendedQuery[operator] = {
...extendedQuery[operator],
...fields,
}
})
})
if (JSON.stringify(query) !== JSON.stringify(extendedQuery)) {
query = extendedQuery
}
}
</script>
<div use:styleable={$component.styles} class="container">
<Provider {actions} data={dataContext}>
{#if !loaded}
<div class="loading">
<ProgressCircle />
</div>
{:else}
{#if $component.emptyState}
<Placeholder />
{:else}
<slot />
{/if}
{#if paginate && internalTable}
<div class="pagination">
<Pagination
page={pageNumber + 1}
{hasPrevPage}
{hasNextPage}
goToPrevPage={prevPage}
goToNextPage={nextPage}
/>
</div>
{/if}
{/if}
</Provider>
</div>
<style>
.container {
display: flex;
flex-direction: column;
justify-content: flex-start;
align-items: stretch;
}
.loading {
display: flex;
flex-direction: row;
justify-content: center;
align-items: center;
height: 100px;
}
.pagination {
display: flex;
flex-direction: row;
justify-content: flex-end;
align-items: center;
margin-top: var(--spacing-xl);
}
</style>

4
packages/client/src/components/app/blocks/CardsBlock.svelte
View File

@ -30,7 +30,7 @@
export let cardButtonOnClick
export let linkColumn
const { API, styleable } = getContext("sdk")
const { fetchDatasourceSchema, styleable } = getContext("sdk")
const context = getContext("context")
const component = getContext("component")
const schemaComponentMap = {
@ -111,7 +111,7 @@
// Load the datasource schema so we can determine column types
const fetchSchema = async dataSource => {
if (dataSource) {
schema = await API.fetchDatasourceSchema(dataSource)
schema = await fetchDatasourceSchema(dataSource)
}
}
</script>

8
packages/client/src/components/app/blocks/TableBlock.svelte
View File

@ -26,7 +26,7 @@
export let titleButtonURL
export let titleButtonPeek
const { API, styleable } = getContext("sdk")
const { fetchDatasourceSchema, styleable } = getContext("sdk")
const context = getContext("context")
const component = getContext("component")
const schemaComponentMap = {
@ -40,6 +40,7 @@
let formId
let dataProviderId
let schema
let schemaLoaded = false
$: fetchSchema(dataSource)
$: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
@ -89,8 +90,9 @@
// Load the datasource schema so we can determine column types
const fetchSchema = async dataSource => {
if (dataSource) {
schema = await API.fetchDatasourceSchema(dataSource)
schema = await fetchDatasourceSchema(dataSource)
}
schemaLoaded = true
}
</script>
@ -134,6 +136,7 @@
</div>
</div>
{/if}
{#if schema}
<BlockComponent
type="dataprovider"
bind:id={dataProviderId}
@ -162,6 +165,7 @@
}}
/>
</BlockComponent>
{/if}
</BlockComponent>
</div>
</Block>

4
packages/client/src/components/app/forms/Form.svelte
View File

@ -10,7 +10,7 @@
export let actionType = "Create"
const context = getContext("context")
const { API } = getContext("sdk")
const { API, fetchDatasourceSchema } = getContext("sdk")
let loaded = false
let schema
@ -61,7 +61,7 @@
// For all other cases, just grab the normal schema
else {
const dataSourceSchema = await API.fetchDatasourceSchema(dataSource)
const dataSourceSchema = await fetchDatasourceSchema(dataSource)
schema = dataSourceSchema || {}
}

2
packages/client/src/sdk.js
View File

@ -9,6 +9,7 @@ import {
import { styleable } from "utils/styleable"
import { linkable } from "utils/linkable"
import { getAction } from "utils/getAction"
import { fetchDatasourceSchema } from "utils/schema.js"
import Provider from "components/context/Provider.svelte"
import { ActionTypes } from "constants"
@ -22,6 +23,7 @@ export default {
styleable,
linkable,
getAction,
fetchDatasourceSchema,
Provider,
ActionTypes,
}

42
packages/client/src/utils/fetch/QueryFetch.js Normal file
View File

@ -0,0 +1,42 @@
import TableFetch from "./TableFetch.js"
import { executeQuery, fetchQueryDefinition } from "api"
import { cloneDeep } from "lodash/fp.js"
export default class ViewFetch extends TableFetch {
SupportsSearch = false
SupportsSort = false
SupportsPagination = false
/**
* Fetches the schema for a view
* @param datasource the view datasource config
* @return {object} the view schema
*/
static async getSchema(datasource) {
if (!datasource?._id) {
return null
}
const definition = await fetchQueryDefinition(datasource._id)
return this.enrichSchema(definition?.schema)
}
/**
* Fetches a single page of data from the remote resource
*/
async getData() {
const { datasource } = this.options
// Set the default query params
let parameters = cloneDeep(datasource?.queryParams || {})
for (let param of datasource?.parameters || {}) {
if (!parameters[param.name]) {
parameters[param.name] = param.default
}
}
const res = await executeQuery({ queryId: datasource?._id, parameters })
return {
rows: res || [],
}
}
}

23
packages/client/src/utils/fetch/RelationshipFetch.js Normal file
View File

@ -0,0 +1,23 @@
import TableFetch from "./TableFetch.js"
import { fetchRelationshipData } from "api"
export default class ViewFetch extends TableFetch {
SupportsSearch = false
SupportsSort = false
SupportsPagination = false
/**
* Fetches a single page of data from the remote resource
*/
async getData() {
const { datasource } = this.options
const res = await fetchRelationshipData({
rowId: datasource?.rowId,
tableId: datasource?.rowTableId,
fieldName: datasource?.fieldName,
})
return {
rows: res || [],
}
}
}

324
packages/client/src/utils/fetch/TableFetch.js Normal file
View File

@ -0,0 +1,324 @@
import { writable, derived, get } from "svelte/store"
import * as API from "api"
import { buildLuceneQuery } from "builder/src/helpers/lucene"
import { fetchTableDefinition } from "api"
/**
* Parent class which handles the implementation of fetching data from an
* internal table or datasource plus.
* For other types of datasource, this class is overridden and extended.
*/
export default class TableFetch {
SupportsSearch = true
SupportsSort = true
SupportsPagination = true
// Config
options = {
datasource: null,
schema: null,
limit: 10,
// Search config
filter: null,
query: null,
// Sorting config
sortColumn: null,
sortOrder: "ascending",
// Pagination config
paginate: true,
}
// State of the fetch
store = writable({
rows: [],
schema: null,
loading: false,
loaded: false,
query: null,
pageNumber: 0,
cursor: null,
cursors: [],
})
/**
* Constructs a new DataFetch instance.
* @param opts the fetch options
*/
constructor(opts) {
this.options = {
...this.options,
...opts,
}
// Bind all functions to properly scope "this"
this.getData = this.getData.bind(this)
this.getInitialData = this.getInitialData.bind(this)
this.refresh = this.refresh.bind(this)
this.update = this.update.bind(this)
this.hasNextPage = this.hasNextPage.bind(this)
this.hasPrevPage = this.hasPrevPage.bind(this)
this.nextPage = this.nextPage.bind(this)
this.prevPage = this.prevPage.bind(this)
// Derive certain properties to return
this.derivedStore = derived(this.store, $store => {
return {
...$store,
hasNextPage: this.hasNextPage($store),
hasPrevPage: this.hasPrevPage($store),
}
})
// Mark as loaded if we have no datasource
if (!this.options.datasource) {
this.store.update($store => ({ ...$store, loaded: true }))
return
}
// Initially fetch data but don't bother waiting for the result
this.getInitialData()
}
/**
* Extend the svelte store subscribe method to that instances of this class
* can be treated like stores
*/
get subscribe() {
return this.derivedStore.subscribe
}
/**
* Fetches a fresh set of data from the server, resetting pagination
*/
async getInitialData() {
const { datasource, filter } = this.options
const tableId = datasource?.tableId
// Ensure table ID exists
if (!tableId) {
return
}
// Ensure schema exists and enrich it
let schema = this.options.schema
if (!schema) {
schema = await this.constructor.getSchema(datasource)
}
if (!schema) {
return
}
// Build the lucene query
let query = this.options.query
if (!query) {
query = buildLuceneQuery(filter)
}
// Update store
this.store.update($store => ({ ...$store, schema, query, loading: true }))
// Actually fetch data
const page = await this.getData()
this.store.update($store => ({
...$store,
loading: false,
loaded: true,
pageNumber: 0,
rows: page.rows,
cursors: page.hasNextPage ? [null, page.cursor] : [null],
}))
}
/**
* Fetches a single page of data from the remote resource
*/
async getData() {
const { datasource, limit, sortColumn, sortOrder, paginate } = this.options
const { tableId } = datasource
const { cursor, schema, query } = get(this.store)
// Work out what sort type to use
const type = schema?.[sortColumn]?.type
const sortType = type === "number" ? "number" : "string"
// Search table
const res = await API.searchTable({
tableId,
query,
limit,
sort: sortColumn,
sortOrder: sortOrder?.toLowerCase() ?? "ascending",
sortType,
paginate,
bookmark: cursor,
})
return {
rows: res?.rows || [],
hasNextPage: res?.hasNextPage || false,
cursor: res?.bookmark || null,
}
}
/**
* Gets the schema definition for a table
* @param datasource the datasource definition
* @return {object} the schema
*/
static async getSchema(datasource) {
if (!datasource?.tableId) {
return null
}
const table = await fetchTableDefinition(datasource.tableId)
return this.enrichSchema(table?.schema)
}
/**
* Enriches the schema and ensures that entries are objects with names
* @param schema the datasource schema
* @return {object} the enriched datasource schema
*/
static enrichSchema(schema) {
if (schema == null) {
return null
}
let enrichedSchema = {}
Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
if (typeof fieldSchema === "string") {
enrichedSchema[fieldName] = {
type: fieldSchema,
name: fieldName,
}
} else {
enrichedSchema[fieldName] = {
...fieldSchema,
name: fieldName,
}
}
})
return enrichedSchema
}
/**
* Resets the data set and updates options
* @param newOptions any new options
*/
async update(newOptions) {
// Check if any settings have actually changed
let refresh = false
const entries = Object.entries(newOptions || {})
for (let [key, value] of entries) {
if (JSON.stringify(value) !== JSON.stringify(this.options[key])) {
refresh = true
break
}
}
if (!refresh) {
return
}
// Assign new options and reload data
this.options = {
...this.options,
...newOptions,
}
await this.getInitialData()
}
/**
* Loads the same page again
*/
async refresh() {
if (get(this.store).loading) {
return
}
const { rows } = await this.getData()
this.store.update($store => ({ ...$store, loading: true }))
this.store.update($store => ({ ...$store, rows, loading: false }))
}
/**
* Determines whether there is a next page of data based on the state of the
* store
* @param state the current store state
* @return {boolean} whether there is a next page of data or not
*/
hasNextPage(state) {
return state.cursors[state.pageNumber + 1] != null
}
/**
* Determines whether there is a previous page of data based on the state of
* the store
* @param state the current store state
* @return {boolean} whether there is a previous page of data or not
*/
hasPrevPage(state) {
return state.pageNumber > 0
}
/**
* Fetches the next page of data
*/
async nextPage() {
const state = get(this.derivedStore)
if (state.loading || !this.options.paginate || !state.hasNextPage) {
return
}
// Fetch next page
const nextCursor = state.cursors[state.pageNumber + 1]
this.store.update($store => ({
...$store,
loading: true,
cursor: nextCursor,
}))
const { rows, hasNextPage, cursor } = await this.getData()
// Update state
this.store.update($store => {
let { cursors, pageNumber } = $store
if (hasNextPage) {
cursors[pageNumber + 2] = cursor
}
return {
...$store,
pageNumber: pageNumber + 1,
rows,
cursors,
loading: false,
}
})
}
/**
* Fetches the previous page of data
*/
async prevPage() {
const state = get(this.derivedStore)
if (state.loading || !this.options.paginate || !state.hasPrevPage) {
return
}
// Fetch previous page
const prevCursor = state.cursors[state.pageNumber - 1]
this.store.update($store => ({
...$store,
loading: true,
cursor: prevCursor,
}))
const { rows } = await this.getData()
// Update state
this.store.update($store => {
return {
...$store,
pageNumber: $store.pageNumber - 1,
rows,
loading: false,
}
})
}
}

34
packages/client/src/utils/fetch/ViewFetch.js Normal file
View File

@ -0,0 +1,34 @@
import TableFetch from "./TableFetch.js"
import { fetchTableDefinition, fetchViewData } from "api"
export default class ViewFetch extends TableFetch {
SupportsSearch = false
SupportsSort = false
SupportsPagination = false
/**
* Fetches the schema for a view
* @param datasource the view datasource config
* @return {object} the view schema
*/
static async getSchema(datasource) {
if (!datasource?.tableId) {
return null
}
const table = await fetchTableDefinition(datasource.tableId)
return this.enrichSchema(table?.views?.[datasource.name]?.schema)
}
/**
* Fetches a single page of data from the remote resource
*/
async getData() {
const { datasource } = this.options
const res = await fetchViewData(datasource)
return {
rows: res || [],
}
}
}
ViewFetch.getSchema()

16
packages/client/src/utils/fetch/fetchData.js Normal file
View File

@ -0,0 +1,16 @@
import TableFetch from "./TableFetch.js"
import ViewFetch from "./ViewFetch.js"
import QueryFetch from "./QueryFetch.js"
import RelationshipFetch from "./RelationshipFetch.js"
const DataFetchMap = {
table: TableFetch,
view: ViewFetch,
query: QueryFetch,
link: RelationshipFetch,
}
export const fetchData = (datasource, options) => {
const Fetch = DataFetchMap[datasource?.type] || TableFetch
return new Fetch({ datasource, ...options })
}

49
packages/client/src/utils/schema.js Normal file
View File

@ -0,0 +1,49 @@
import TableFetch from "./fetch/TableFetch.js"
import ViewFetch from "./fetch/ViewFetch.js"
import QueryFetch from "./fetch/QueryFetch.js"
/**
* Fetches the schema of any kind of datasource.
*/
export const fetchDatasourceSchema = async datasource => {
const type = datasource?.type
// Nested providers should already have exposed their own schema
if (type === "provider") {
return datasource.value?.schema
}
// Field sources have their schema statically defined
if (type === "field") {
if (datasource.fieldType === "attachment") {
return {
url: {
type: "string",
},
name: {
type: "string",
},
}
} else if (datasource.fieldType === "array") {
return {
value: {
type: "string",
},
}
}
}
// All normal datasource schema can use their corresponsing implementations
// in the data fetch classes
if (type === "table" || type === "link") {
return TableFetch.getSchema(datasource)
}
if (type === "view") {
return ViewFetch.getSchema(datasource)
}
if (type === "query") {
return QueryFetch.getSchema(datasource)
}
return null
}