MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve typing around in-memory search.

This commit is contained in:
Sam Rose 2024-09-06 15:03:17 +01:00
parent 7e5f199f3b
commit 3c58a593f9
No known key found for this signature in database
4 changed files with 158 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/server/src/integrations/googlesheets.ts
View File

@ -580,7 +580,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
let response = []
for (let row of filtered) {
response.push(
this.buildRowObject(headerValues, row.toObject(), row._rowNumber)
this.buildRowObject(headerValues, row.toObject(), row["_rowNumber"])
)
}

184
packages/server/src/integrations/tests/googlesheets.spec.ts
View File

@ -1,36 +1,22 @@
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet"
import nock from "nock"
import { structures } from "@budibase/backend-core/tests"
import TestConfiguration from "../../tests/utilities/TestConfiguration"
import { GoogleSheetsConfig, GoogleSheetsIntegration } from "../googlesheets"
import {
Datasource,
FieldType,
SourceName,
Table,
TableSchema,
TableSourceType,
} from "@budibase/types"
import { generateDatasourceID } from "../../db/utils"
import { access } from "node:fs"
import { GoogleSheetsMock } from "./utils/googlesheets"
describe("Google Sheets Integration", () => {
const config = new TestConfiguration()
let integration: GoogleSheetsIntegration
let cleanupEnv: () => void
let table: Table
let datasource: Datasource
const datasourceConfig: GoogleSheetsConfig = {
spreadsheetId: "randomId",
auth: {
appId: "appId",
accessToken: "accessToken",
refreshToken: "refreshToken",
},
}
let mock: GoogleSheetsMock
beforeAll(async () => {
cleanupEnv = setCoreEnv({
@ -38,11 +24,20 @@ describe("Google Sheets Integration", () => {
GOOGLE_CLIENT_SECRET: "test",
})
await config.init()
datasource = await config.api.datasource.create({
name: "Test Datasource",
type: "datasource",
source: SourceName.GOOGLE_SHEETS,
config: datasourceConfig,
config: {
spreadsheetId: "randomId",
auth: {
appId: "appId",
accessToken: "accessToken",
refreshToken: "refreshToken",
},
},
})
})
@ -52,139 +47,40 @@ describe("Google Sheets Integration", () => {
})
beforeEach(async () => {
await config.init()
integration = new GoogleSheetsIntegration(datasourceConfig)
table = await config.api.table.save({
name: "Test Table",
type: "table",
sourceId: generateDatasourceID(),
sourceType: TableSourceType.EXTERNAL,
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
type: "string",
},
},
description: {
name: "description",
type: FieldType.STRING,
constraints: {
type: "string",
},
},
},
})
nock.cleanAll()
nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, {
grant_type: "client_credentials",
client_id: "your-client-id",
client_secret: "your-client-secret",
})
mock = GoogleSheetsMock.forDatasource(datasource)
mock.mockAuth()
})
function createBasicTable(name: string, columns: string[]): Table {
return {
type: "table",
name,
sourceId: generateDatasourceID(),
sourceType: TableSourceType.EXTERNAL,
schema: {
...columns.reduce((p, c) => {
p[c] = {
name: c,
describe("create", () => {
it("creates a new table", async () => {
nock("https://sheets.googleapis.com/", {
reqheaders: { authorization: "Bearer test" },
})
.get("/v4/spreadsheets/randomId/")
.reply(200, {})
const table = await config.api.table.save({
name: "Test Table",
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
type: "string",
},
}
return p
}, {} as TableSchema),
},
}
}
function createSheet({
headerValues,
}: {
headerValues: string[]
}): GoogleSpreadsheetWorksheet {
return {
// to ignore the unmapped fields
...({} as any),
loadHeaderRow: jest.fn(),
headerValues,
setHeaderRow: jest.fn(),
}
}
describe("update table", () => {
it("adding a new field will be adding a new header row", async () => {
await config.doInContext(structures.uuid(), async () => {
const tableColumns = ["name", "description", "new field"]
const table = createBasicTable(structures.uuid(), tableColumns)
const sheet = createSheet({ headerValues: ["name", "description"] })
sheetsByTitle[table.name] = sheet
await integration.updateTable(table)
expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1)
expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1)
expect(sheet.setHeaderRow).toHaveBeenCalledWith(tableColumns)
})
})
it("removing an existing field will remove the header from the google sheet", async () => {
const sheet = await config.doInContext(structures.uuid(), async () => {
const tableColumns = ["name"]
const table = createBasicTable(structures.uuid(), tableColumns)
const sheet = createSheet({
headerValues: ["name", "description", "location"],
})
sheetsByTitle[table.name] = sheet
await integration.updateTable(table)
return sheet
})
expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1)
expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1)
expect(sheet.setHeaderRow).toHaveBeenCalledWith([
"name",
"description",
"location",
])
})
})
describe("getTableNames", () => {
it("can fetch table names", async () => {
await config.doInContext(structures.uuid(), async () => {
const sheetNames: string[] = []
for (let i = 0; i < 5; i++) {
const sheet = createSheet({ headerValues: [] })
sheetsByIndex.push(sheet)
sheetNames.push(sheet.title)
}
const res = await integration.getTableNames()
expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1)
expect(res).toEqual(sheetNames)
})
})
})
describe("testConnection", () => {
it("can test successful connections", async () => {
await config.doInContext(structures.uuid(), async () => {
const res = await integration.testConnection()
expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1)
expect(res).toEqual({ connected: true })
},
description: {
name: "description",
type: FieldType.STRING,
constraints: {
type: "string",
},
},
},
})
})
})

93
packages/server/src/integrations/tests/utils/googlesheets.ts Normal file
View File

@ -0,0 +1,93 @@
import { Datasource } from "@budibase/types"
import nock from "nock"
import { GoogleSheetsConfig } from "../../googlesheets"
interface ErrorValue {
type: string
message: string
}
interface ExtendedValue {
stringValue?: string
numberValue?: number
boolValue?: boolean
formulaValue?: string
errorValue?: ErrorValue
}
interface CellData {
userEnteredValue: ExtendedValue
}
interface RowData {
values: CellData[]
}
interface GridData {
startRow: number
startColumn: number
rowData: RowData[]
}
interface Sheet {
properties: {
sheetId: string
title: string
}
data: GridData[]
}
interface Spreadsheet {
spreadsheetId: string
sheets: Sheet[]
}
export class GoogleSheetsMock {
private config: GoogleSheetsConfig
private sheet: Spreadsheet
static forDatasource(datasource: Datasource): GoogleSheetsMock {
return new GoogleSheetsMock(datasource.config as GoogleSheetsConfig)
}
private constructor(config: GoogleSheetsConfig) {
this.config = config
this.sheet = {
spreadsheetId: config.spreadsheetId,
sheets: [],
}
}
init() {
nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, {
grant_type: "client_credentials",
client_id: "your-client-id",
client_secret: "your-client-secret",
})
nock("https://oauth2.googleapis.com/")
.post("/token", {
client_id: "test",
client_secret: "test",
grant_type: "refresh_token",
refresh_token: "refreshToken",
})
.reply(200, {
access_token: "test",
expires_in: 3600,
token_type: "Bearer",
scopes: "https://www.googleapis.com/auth/spreadsheets",
})
nock("https://sheets.googleapis.com/", {
reqheaders: { authorization: "Bearer test" },
})
.get("/v4/spreadsheets/randomId/")
.reply(200, {})
nock("https://sheets.googleapis.com/", {
reqheaders: { authorization: "Bearer test" },
})
.post(`/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`)
.reply(200, () => {})
}
}

53
packages/shared-core/src/filters.ts
View File

@ -448,10 +448,10 @@ export function fixupFilterArrays(filters: SearchFilters) {
return filters
}
export const search = (
docs: Record<string, any>[],
export function search<T>(
docs: Record<string, T>[],
query: RowSearchParams
): SearchResponse<Record<string, any>> => {
): SearchResponse<Record<string, T>> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
@ -475,11 +475,11 @@ export const search = (
* from custom doc type e.g. Google Sheets
*
*/
export const runQuery = (
docs: Record<string, any>[],
export function runQuery<T extends Record<string, any>>(
docs: T[],
query: SearchFilters,
findInDoc: Function = deepGet
) => {
findInDoc: (obj: T, key: string) => any = deepGet
): T[] {
if (!docs || !Array.isArray(docs)) {
return []
}
@ -502,7 +502,7 @@ export const runQuery = (
type: SearchFilterOperator,
test: (docValue: any, testValue: any) => boolean
) =>
(doc: Record<string, any>) => {
(doc: T) => {
for (const [key, testValue] of Object.entries(query[type] || {})) {
const valueToCheck = isLogicalSearchOperator(type)
? doc
@ -749,11 +749,8 @@ export const runQuery = (
}
)
const docMatch = (doc: Record<string, any>) => {
const filterFunctions: Record<
SearchFilterOperator,
(doc: Record<string, any>) => boolean
> = {
const docMatch = (doc: T) => {
const filterFunctions: Record<SearchFilterOperator, (doc: T) => boolean> = {
string: stringMatch,
fuzzy: fuzzyMatch,
range: rangeMatch,
@ -797,12 +794,12 @@ export const runQuery = (
* @param sortOrder the sort order ("ascending" or "descending")
* @param sortType the type of sort ("string" or "number")
*/
export const sort = (
docs: any[],
sort: string,
export function sort<T extends Record<string, any>>(
docs: T[],
sort: keyof T,
sortOrder: SortOrder,
sortType = SortType.STRING
) => {
): T[] {
if (!sort || !sortOrder || !sortType) {
return docs
}
@ -817,19 +814,17 @@ export const sort = (
return parseFloat(x)
}
return docs
.slice()
.sort((a: { [x: string]: any }, b: { [x: string]: any }) => {
const colA = parse(a[sort])
const colB = parse(b[sort])
return docs.slice().sort((a, b) => {
const colA = parse(a[sort])
const colB = parse(b[sort])
const result = colB == null || colA > colB ? 1 : -1
if (sortOrder.toLowerCase() === "descending") {
return result * -1
}
const result = colB == null || colA > colB ? 1 : -1
if (sortOrder.toLowerCase() === "descending") {
return result * -1
}
return result
})
return result
})
}
/**
@ -838,7 +833,7 @@ export const sort = (
* @param docs the data
* @param limit the number of docs to limit to
*/
export const limit = (docs: any[], limit: string) => {
export function limit<T>(docs: T[], limit: string): T[] {
const numLimit = parseFloat(limit)
if (isNaN(numLimit)) {
return docs