diff --git a/packages/data-utils/.gitignore b/packages/data-utils/.gitignore new file mode 100644 index 0000000000..c2658d7d1b --- /dev/null +++ b/packages/data-utils/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/packages/data-utils/package.json b/packages/data-utils/package.json new file mode 100644 index 0000000000..26d4a25232 --- /dev/null +++ b/packages/data-utils/package.json @@ -0,0 +1,12 @@ +{ + "name": "@budibase/data-utils", + "version": "0.0.1", + "description": "Shared data utils", + "main": "src/index.ts", + "author": "Budibase", + "license": "GPL-3.0", + "private": true, + "devDependencies": { + "typescript": "4.7.3" + } +} diff --git a/packages/data-utils/src/constants.ts b/packages/data-utils/src/constants.ts new file mode 100644 index 0000000000..35b02d1e15 --- /dev/null +++ b/packages/data-utils/src/constants.ts @@ -0,0 +1,69 @@ +export const OperatorOptions = { + Equals: { + value: "equal", + label: "Equals", + }, + NotEquals: { + value: "notEqual", + label: "Not equals", + }, + Empty: { + value: "empty", + label: "Is empty", + }, + NotEmpty: { + value: "notEmpty", + label: "Is not empty", + }, + StartsWith: { + value: "string", + label: "Starts with", + }, + Like: { + value: "fuzzy", + label: "Like", + }, + MoreThan: { + value: "rangeLow", + label: "More than or equal to", + }, + LessThan: { + value: "rangeHigh", + label: "Less than or equal to", + }, + Contains: { + value: "contains", + label: "Contains", + }, + NotContains: { + value: "notContains", + label: "Does not contain", + }, + In: { + value: "oneOf", + label: "Is in", + }, + ContainsAny: { + value: "containsAny", + label: "Has any", + }, +} + +export const SqlNumberTypeRangeMap = { + integer: { + max: 2147483647, + min: -2147483648, + }, + int: { + max: 2147483647, + min: -2147483648, + }, + smallint: { + max: 32767, + min: -32768, + }, + mediumint: { + max: 8388607, + min: -8388608, + }, +} diff --git a/packages/data-utils/src/filters.ts b/packages/data-utils/src/filters.ts new file mode 100644 index 0000000000..fa97f5a647 --- /dev/null +++ b/packages/data-utils/src/filters.ts @@ -0,0 +1,427 @@ +import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" + +const HBS_REGEX = /{{([^{].*?)}}/g + +/** + * Returns the valid operator options for a certain data type + * @param type the data type + */ +export const getValidOperatorsForType = ( + type: string, + field: string, + datasource: { tableId: string | string[]; type: string } +) => { + const Op = OperatorOptions + const stringOps = [ + Op.Equals, + Op.NotEquals, + Op.StartsWith, + Op.Like, + Op.Empty, + Op.NotEmpty, + Op.In, + ] + const numOps = [ + Op.Equals, + Op.NotEquals, + Op.MoreThan, + Op.LessThan, + Op.Empty, + Op.NotEmpty, + Op.In, + ] + let ops: any[] = [] + if (type === "string") { + ops = stringOps + } else if (type === "number") { + ops = numOps + } else if (type === "options") { + ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In] + } else if (type === "array") { + ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny] + } else if (type === "boolean") { + ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty] + } else if (type === "longform") { + ops = stringOps + } else if (type === "datetime") { + ops = numOps + } else if (type === "formula") { + ops = stringOps.concat([Op.MoreThan, Op.LessThan]) + } + + // Filter out "like" for internal tables + const externalTable = datasource?.tableId?.includes("datasource_plus") + if (datasource?.type === "table" && !externalTable) { + ops = ops.filter(x => x !== Op.Like) + } + + // Only allow equal/not equal for _id in SQL tables + if (field === "_id" && externalTable) { + ops = [Op.Equals, Op.NotEquals] + } + + return ops +} + +/** + * Operators which do not support empty strings as values + */ +export const NoEmptyFilterStrings = [ + OperatorOptions.StartsWith.value, + OperatorOptions.Like.value, + OperatorOptions.Equals.value, + OperatorOptions.NotEquals.value, + OperatorOptions.Contains.value, + OperatorOptions.NotContains.value, +] + +/** + * Removes any fields that contain empty strings that would cause inconsistent + * behaviour with how backend tables are filtered (no value means no filter). + */ +const cleanupQuery = (query: { [x: string]: { [x: string]: any } }) => { + if (!query) { + return query + } + for (let filterField of NoEmptyFilterStrings) { + if (!query[filterField]) { + continue + } + for (let [key, value] of Object.entries(query[filterField])) { + if (value == null || value === "") { + delete query[filterField][key] + } + } + } + return query +} + +/** + * Removes a numeric prefix on field names designed to give fields uniqueness + */ +const removeKeyNumbering = (key: string) => { + if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) { + const parts = key.split(":") + parts.shift() + return parts.join(":") + } else { + return key + } +} + +/** + * Builds a lucene JSON query from the filter structure generated in the builder + * @param filter the builder filter structure + */ +export const buildLuceneQuery = (filter: any[]) => { + let query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + if (Array.isArray(filter)) { + filter.forEach(expression => { + let { operator, field, type, value, externalType } = expression + const isHbs = + typeof value === "string" && value.match(HBS_REGEX)?.length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if ( + type === "datetime" && + !isHbs && + operator !== "empty" && + operator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string") { + if (operator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else if (!isHbs) { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes(operator) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.startsWith("range")) { + const minint = + SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field].low = value + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field].high = value + } + } else if (query[operator]) { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (operator === "equal" && value === false) { + query.notEqual[field] = true + } else if (operator === "notEqual" && value === false) { + query.equal[field] = true + } else { + query[operator][field] = value + } + } else { + query[operator][field] = value + } + } + }) + } + return query +} + +const deepGet = (obj: { [x: string]: any }, key: string) => { + if (!obj || !key) { + return null + } + if (Object.prototype.hasOwnProperty.call(obj, key)) { + return obj[key] + } + const split = key.split(".") + for (let i = 0; i < split.length; i++) { + obj = obj?.[split[i]] + } + return obj +} + +/** + * Performs a client-side lucene search on an array of data + * @param docs the data + * @param query the JSON lucene query + */ +export const runLuceneQuery = ( + docs: any[], + query?: { [x: string]: any; sheet?: string } +) => { + if (!docs || !Array.isArray(docs)) { + return [] + } + if (!query) { + return docs + } + + // Make query consistent first + query = cleanupQuery(query) + + // Iterates over a set of filters and evaluates a fail function against a doc + const match = + ( + type: string, + failFn: { + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any): boolean + (docValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): boolean + (docValue: any, testValue: any): any + (arg0: any, arg1: unknown): any + } + ) => + (doc: any) => { + const filters = Object.entries(query[type] || {}) + for (let i = 0; i < filters.length; i++) { + const [key, testValue] = filters[i] + const docValue = deepGet(doc, removeKeyNumbering(key)) + if (failFn(docValue, testValue)) { + return false + } + } + return true + } + + // Process a string match (fails if the value does not start with the string) + const stringMatch = match("string", (docValue: string, testValue: string) => { + return ( + !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) + ) + }) + + // Process a fuzzy match (treat the same as starts with when running locally) + const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => { + return ( + !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) + ) + }) + + // Process a range match + const rangeMatch = match( + "range", + ( + docValue: string | number | null, + testValue: { low: number; high: number } + ) => { + return ( + docValue == null || + docValue === "" || + docValue < testValue.low || + docValue > testValue.high + ) + } + ) + + // Process an equal match (fails if the value is different) + const equalMatch = match( + "equal", + (docValue: any, testValue: string | null) => { + return testValue != null && testValue !== "" && docValue !== testValue + } + ) + + // Process a not-equal match (fails if the value is the same) + const notEqualMatch = match( + "notEqual", + (docValue: any, testValue: string | null) => { + return testValue != null && testValue !== "" && docValue === testValue + } + ) + + // Process an empty match (fails if the value is not empty) + const emptyMatch = match("empty", (docValue: string | null) => { + return docValue != null && docValue !== "" + }) + + // Process a not-empty match (fails is the value is empty) + const notEmptyMatch = match("notEmpty", (docValue: string | null) => { + return docValue == null || docValue === "" + }) + + // Process an includes match (fails if the value is not included) + const oneOf = match("oneOf", (docValue: any, testValue: string[]) => { + if (typeof testValue === "string") { + testValue = testValue.split(",") + if (typeof docValue === "number") { + testValue = testValue.map((item: string) => parseFloat(item)) + } + } + return !testValue?.includes(docValue) + }) + + const containsAny = match( + "containsAny", + (docValue: string | any[], testValue: any) => { + return !docValue?.includes(...testValue) + } + ) + + const contains = match( + "contains", + (docValue: string | any[], testValue: any[]) => { + return !testValue?.every((item: any) => docValue?.includes(item)) + } + ) + + const notContains = match( + "notContains", + (docValue: string | any[], testValue: any[]) => { + return testValue?.every((item: any) => docValue?.includes(item)) + } + ) + + // Match a document against all criteria + const docMatch = (doc: any) => { + return ( + stringMatch(doc) && + fuzzyMatch(doc) && + rangeMatch(doc) && + equalMatch(doc) && + notEqualMatch(doc) && + emptyMatch(doc) && + notEmptyMatch(doc) && + oneOf(doc) && + contains(doc) && + containsAny(doc) && + notContains(doc) + ) + } + + // Process all docs + return docs.filter(docMatch) +} + +/** + * Performs a client-side sort from the equivalent server-side lucene sort + * parameters. + * @param docs the data + * @param sort the sort column + * @param sortOrder the sort order ("ascending" or "descending") + * @param sortType the type of sort ("string" or "number") + */ +export const luceneSort = ( + docs: any[], + sort: string | number, + sortOrder: string, + sortType = "string" +) => { + if (!sort || !sortOrder || !sortType) { + return docs + } + const parse = + sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x) + return docs + .slice() + .sort((a: { [x: string]: any }, b: { [x: string]: any }) => { + const colA = parse(a[sort]) + const colB = parse(b[sort]) + if (sortOrder === "Descending") { + return colA > colB ? -1 : 1 + } else { + return colA > colB ? 1 : -1 + } + }) +} + +/** + * Limits the specified docs to the specified number of rows from the equivalent + * server-side lucene limit parameters. + * @param docs the data + * @param limit the number of docs to limit to + */ +export const luceneLimit = (docs: string | any[], limit: string) => { + const numLimit = parseFloat(limit) + if (isNaN(numLimit)) { + return docs + } + return docs.slice(0, numLimit) +} diff --git a/packages/data-utils/src/index.ts b/packages/data-utils/src/index.ts new file mode 100644 index 0000000000..5506ead439 --- /dev/null +++ b/packages/data-utils/src/index.ts @@ -0,0 +1,2 @@ +export * from "./constants" +export * from "./filters" diff --git a/packages/data-utils/tsconfig.build.json b/packages/data-utils/tsconfig.build.json new file mode 100644 index 0000000000..12f8255a7c --- /dev/null +++ b/packages/data-utils/tsconfig.build.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "es6", + "module": "commonjs", + "lib": ["es2020"], + "strict": true, + "noImplicitAny": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "incremental": true, + "sourceMap": true, + "declaration": true, + "types": [ "node", "jest" ], + "outDir": "dist", + "skipLibCheck": true + }, + "include": [ + "**/*.js", + "**/*.ts", + "package.json" + ], + "exclude": [ + "node_modules", + "dist", + "**/*.spec.ts", + "**/*.spec.js", + "__mocks__" + ] +} \ No newline at end of file diff --git a/packages/data-utils/tsconfig.json b/packages/data-utils/tsconfig.json new file mode 100644 index 0000000000..4fd2295642 --- /dev/null +++ b/packages/data-utils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "composite": true, + "baseUrl": "." + }, + "exclude": ["node_modules", "dist"] +} diff --git a/packages/data-utils/yarn.lock b/packages/data-utils/yarn.lock new file mode 100644 index 0000000000..6774dcd692 --- /dev/null +++ b/packages/data-utils/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +typescript@4.7.3: + version "4.7.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d" + integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA== diff --git a/packages/server/package.json b/packages/server/package.json index ed6ca35fd1..f84339e167 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -45,6 +45,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "2.3.18-alpha.29", "@budibase/client": "2.3.18-alpha.29", + "@budibase/data-utils": "0.0.1", "@budibase/pro": "2.3.18-alpha.29", "@budibase/string-templates": "2.3.18-alpha.29", "@budibase/types": "2.3.18-alpha.29", diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6f202f9c3a..69df1f7289 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -2,8 +2,11 @@ import { DatasourceFieldType, DatasourcePlus, Integration, + PaginationJson, QueryJson, QueryType, + SearchFilters, + SortJson, Table, TableSchema, } from "@budibase/types" @@ -13,6 +16,7 @@ import { DataSourceOperation, FieldTypes } from "../constants" import { GoogleSpreadsheet } from "google-spreadsheet" import fetch from "node-fetch" import { configs, HTTPError } from "@budibase/backend-core" +import { runLuceneQuery } from "@budibase/data-utils" interface GoogleSheetsConfig { spreadsheetId: string @@ -237,7 +241,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { const handlers = { [DataSourceOperation.CREATE]: () => this.create({ sheet, row: json.body }), - [DataSourceOperation.READ]: () => this.read({ sheet }), + [DataSourceOperation.READ]: () => this.read({ ...json, sheet }), [DataSourceOperation.UPDATE]: () => this.update({ // exclude the header row and zero index @@ -345,14 +349,20 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - async read(query: { sheet: string }) { + async read(query: { + sheet: string + filters?: SearchFilters + sort?: SortJson + paginate?: PaginationJson + }) { try { await this.connect() const sheet = this.client.sheetsByTitle[query.sheet] const rows = await sheet.getRows() + const filtered = runLuceneQuery(rows, query.filters) const headerValues = sheet.headerValues const response = [] - for (let row of rows) { + for (let row of filtered) { response.push( this.buildRowObject(headerValues, row._rawData, row._rowNumber) ) diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index dba0d6328a..937d7adb8a 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -9,6 +9,7 @@ "@budibase/types": ["../types/src"], "@budibase/backend-core": ["../backend-core/src"], "@budibase/backend-core/*": ["../backend-core/*"], + "@budibase/data-utils": ["../data-utils/src"], "@budibase/pro": ["../../../budibase-pro/packages/pro/src"] } }, @@ -19,15 +20,9 @@ "references": [ { "path": "../types" }, { "path": "../backend-core" }, + { "path": "../data-utils" }, { "path": "../../../budibase-pro/packages/pro" } ], - "include": [ - "src/**/*", - "specs", - "package.json" - ], - "exclude": [ - "node_modules", - "dist" - ] -} \ No newline at end of file + "include": ["src/**/*", "specs", "package.json"], + "exclude": ["node_modules", "dist"] +}