MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Type datafetch

This commit is contained in:
Adria Navarro 2024-12-31 15:43:08 +01:00
parent 913cefaf17
commit ecfc248e60
4 changed files with 128 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
packages/frontend-core/src/fetch/DataFetch.js → packages/frontend-core/src/fetch/DataFetch.ts
View File

@ -1,25 +1,86 @@
import { writable, derived, get } from "svelte/store"
import { writable, derived, get, Writable, Readable } from "svelte/store"
import { cloneDeep } from "lodash/fp"
import { QueryUtils } from "../utils"
import { convertJSONSchemaToTableSchema } from "../utils/json"
import { FieldType, SortOrder, SortType } from "@budibase/types"
import {
FieldType,
LegacyFilter,
SearchFilters,
SortOrder,
SortType,
Table,
TableSchema,
UIDatasource,
UIFetchAPI,
UIRow,
UISearchFilter,
} from "@budibase/types"
const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
interface DataFetchStore {
rows: UIRow[]
info: null
schema: TableSchema | null
loading: boolean
loaded: boolean
query: SearchFilters | null
pageNumber: number
cursor: null
cursors: any[]
resetKey: number
error: null
}
interface DataFetchDerivedStore extends DataFetchStore {
hasNextPage: boolean
hasPrevPage: boolean
supportsSearch: boolean
supportsSort: boolean
supportsPagination: boolean
}
/**
* Parent class which handles the implementation of fetching data from an
* internal table or datasource plus.
* For other types of datasource, this class is overridden and extended.
*/
export default class DataFetch {
export default abstract class DataFetch {
API: UIFetchAPI
features: {
supportsSearch: boolean
supportsSort: boolean
supportsPagination: boolean
}
options: {
datasource: UIDatasource | null
limit: number
// Search config
filter: UISearchFilter | LegacyFilter[] | null
query: SearchFilters | null
// Sorting config
sortColumn: string | null
sortOrder: SortOrder
sortType: SortType | null
// Pagination config
paginate: boolean
// Client side feature customisation
clientSideSearching: boolean
clientSideSorting: boolean
clientSideLimiting: boolean
}
store: Writable<DataFetchStore>
derivedStore: Readable<DataFetchDerivedStore>
/**
* Constructs a new DataFetch instance.
* @param opts the fetch options
*/
constructor(opts) {
// API client
this.API = null
constructor(opts: {
API: UIFetchAPI
datasource?: UIDatasource
options?: {}
}) {
// Feature flags
this.features = {
supportsSearch: false,
@ -118,7 +179,10 @@ export default class DataFetch {
/**
* Gets the default sort column for this datasource
*/
getDefaultSortColumn(definition, schema) {
getDefaultSortColumn(
definition: { primaryDisplay?: string } | null,
schema: Record<string, any>
) {
if (definition?.primaryDisplay && schema[definition.primaryDisplay]) {
return definition.primaryDisplay
} else {
@ -144,7 +208,7 @@ export default class DataFetch {
}
// Fetch and enrich schema
let schema = this.getSchema(datasource, definition)
let schema = this.getSchema(datasource, definition) ?? null
schema = this.enrichSchema(schema)
if (!schema) {
return
@ -172,7 +236,7 @@ export default class DataFetch {
if (
fieldSchema?.type === FieldType.NUMBER ||
fieldSchema?.type === FieldType.BIGINT ||
fieldSchema?.calculationType
("calculationType" in fieldSchema && fieldSchema?.calculationType)
) {
this.options.sortType = SortType.NUMBER
}
@ -185,7 +249,7 @@ export default class DataFetch {
// Build the query
let query = this.options.query
if (!query) {
query = buildQuery(filter)
query = buildQuery(filter ?? undefined)
}
// Update store
@ -239,7 +303,7 @@ export default class DataFetch {
// If we don't support sorting, do a client-side sort
if (!this.features.supportsSort && clientSideSorting) {
rows = sort(rows, sortColumn, sortOrder, sortType)
rows = sort(rows, sortColumn as any, sortOrder, sortType)
}
// If we don't support pagination, do a client-side limit
@ -256,18 +320,13 @@ export default class DataFetch {
}
}
/**
* Fetches a single page of data from the remote resource.
* Must be overridden by a datasource specific child class.
*/
async getData() {
return {
rows: [],
info: null,
hasNextPage: false,
cursor: null,
}
}
abstract getData(): Promise<{
rows: UIRow[]
info: any
hasNextPage: boolean
cursor: any
error: any
}>
/**
* Gets the definition for this datasource.
@ -275,13 +334,13 @@ export default class DataFetch {
* @param datasource
* @return {object} the definition
*/
async getDefinition(datasource) {
async getDefinition(datasource: UIDatasource | null) {
if (!datasource?.tableId) {
return null
}
try {
return await this.API.fetchTableDefinition(datasource.tableId)
} catch (error) {
} catch (error: any) {
this.store.update(state => ({
...state,
error,
@ -293,11 +352,11 @@ export default class DataFetch {
/**
* Gets the schema definition for a datasource.
* Defaults to getting the "schema" property of the definition.
* @param datasource the datasource
* @param _datasource the datasource
* @param definition the datasource definition
* @return {object} the schema
*/
getSchema(datasource, definition) {
getSchema(_datasource: UIDatasource | null, definition: Table | null) {
return definition?.schema
}
@ -307,44 +366,48 @@ export default class DataFetch {
* @param schema the datasource schema
* @return {object} the enriched datasource schema
*/
enrichSchema(schema) {
enrichSchema(schema: TableSchema | null): TableSchema | null {
if (schema == null) {
return null
}
// Check for any JSON fields so we can add any top level properties
let jsonAdditions = {}
Object.keys(schema).forEach(fieldKey => {
let jsonAdditions: Record<string, { type: string; nestedJSON: true }> = {}
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey]
if (fieldSchema?.type === FieldType.JSON) {
const jsonSchema = convertJSONSchemaToTableSchema(fieldSchema, {
squashObjects: true,
})
Object.keys(jsonSchema).forEach(jsonKey => {
jsonAdditions[`${fieldKey}.${jsonKey}`] = {
type: jsonSchema[jsonKey].type,
nestedJSON: true,
}) as Record<string, { type: string }> | null // TODO: remove when convertJSONSchemaToTableSchema is typed
if (jsonSchema) {
for (const jsonKey of Object.keys(jsonSchema)) {
jsonAdditions[`${fieldKey}.${jsonKey}`] = {
type: jsonSchema[jsonKey].type,
nestedJSON: true,
}
}
})
}
}
})
schema = { ...schema, ...jsonAdditions }
}
// Ensure schema is in the correct structure
let enrichedSchema = {}
Object.entries(schema).forEach(([fieldName, fieldSchema]) => {
if (typeof fieldSchema === "string") {
enrichedSchema[fieldName] = {
type: fieldSchema,
name: fieldName,
}
} else {
enrichedSchema[fieldName] = {
...fieldSchema,
name: fieldName,
let enrichedSchema: TableSchema = {}
Object.entries({ ...schema, ...jsonAdditions }).forEach(
([fieldName, fieldSchema]) => {
if (typeof fieldSchema === "string") {
enrichedSchema[fieldName] = {
type: fieldSchema,
name: fieldName,
}
} else {
enrichedSchema[fieldName] = {
...fieldSchema,
type: fieldSchema.type as any, // TODO: check type union definition conflicts
name: fieldName,
}
}
}
})
)
return enrichedSchema
}
@ -353,7 +416,7 @@ export default class DataFetch {
* Determine the feature flag for this datasource definition
* @param definition
*/
determineFeatureFlags(_definition) {
determineFeatureFlags(_definition: Table | null) {
return {
supportsSearch: false,
supportsSort: false,
@ -365,12 +428,11 @@ export default class DataFetch {
* Resets the data set and updates options
* @param newOptions any new options
*/
async update(newOptions) {
async update(newOptions: any) {
// Check if any settings have actually changed
let refresh = false
const entries = Object.entries(newOptions || {})
for (let [key, value] of entries) {
const oldVal = this.options[key] == null ? null : this.options[key]
for (const [key, value] of Object.entries(newOptions || {})) {
const oldVal = this.options[key as keyof typeof this.options] ?? null
const newVal = value == null ? null : value
if (JSON.stringify(newVal) !== JSON.stringify(oldVal)) {
refresh = true
@ -437,7 +499,7 @@ export default class DataFetch {
* @param state the current store state
* @return {boolean} whether there is a next page of data or not
*/
hasNextPage(state) {
hasNextPage(state: DataFetchStore): boolean {
return state.cursors[state.pageNumber + 1] != null
}
@ -447,7 +509,7 @@ export default class DataFetch {
* @param state the current store state
* @return {boolean} whether there is a previous page of data or not
*/
hasPrevPage(state) {
hasPrevPage(state: { pageNumber: number }): boolean {
return state.pageNumber > 0
}

4
packages/frontend-core/src/fetch/index.ts
View File

@ -10,7 +10,7 @@ import UserFetch from "./UserFetch.js"
import GroupUserFetch from "./GroupUserFetch.js"
import CustomFetch from "./CustomFetch.js"
import QueryArrayFetch from "./QueryArrayFetch.js"
import { UIDatasource, UIFetchAPI } from "@budibase/types"
import { Table, UIDatasource, UIFetchAPI } from "@budibase/types"
const DataFetchMap = {
table: TableFetch,
@ -80,7 +80,7 @@ export const getDatasourceSchema = ({
}: {
API: UIFetchAPI
datasource: UIDatasource
definition: {}
definition: Table
}) => {
const instance = createEmptyFetchInstance({ API, datasource })
return instance?.getSchema(datasource, definition)

8
packages/shared-core/src/filters.ts
View File

@ -552,7 +552,7 @@ export function search<T extends Record<string, any>>(
*/
export function runQuery<T extends Record<string, any>>(
docs: T[],
query: SearchFilters
query: SearchFilters | null
): T[] {
if (!docs || !Array.isArray(docs)) {
return []
@ -876,7 +876,7 @@ export function sort<T extends Record<string, any>>(
docs: T[],
sort: keyof T,
sortOrder: SortOrder,
sortType = SortType.STRING
sortType: SortType | null = SortType.STRING
): T[] {
if (!sort || !sortOrder || !sortType) {
return docs
@ -911,8 +911,8 @@ export function sort<T extends Record<string, any>>(
* @param docs the data
* @param limit the number of docs to limit to
*/
export function limit<T>(docs: T[], limit: string): T[] {
const numLimit = parseFloat(limit)
export function limit<T>(docs: T[], limit: string | number): T[] {
const numLimit = typeof limit === "number" ? limit : parseFloat(limit)
if (isNaN(numLimit)) {
return docs
}

2
packages/types/src/ui/stores/grid/fetch.ts
View File

@ -1,12 +1,14 @@
import {
Row,
SortOrder,
Table,
UIDatasource,
UILegacyFilter,
UISearchFilter,
} from "@budibase/types"
export interface UIFetchAPI {
fetchTableDefinition(tableId: string): Promise<Table>
definition: UIDatasource
getInitialData: () => Promise<void>