MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fetch schema tests.

This commit is contained in:
Sam Rose 2024-09-19 16:48:50 +01:00
parent 40e654f515
commit aecd4f9e4d
No known key found for this signature in database
5 changed files with 187 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/server/package.json
View File

@ -80,7 +80,7 @@
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2",
"isolated-vm": "^4.7.2",
"jimp": "0.22.12",

17
packages/server/src/integrations/googlesheets.ts
View File

@ -339,9 +339,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
const tables: Record<string, Table> = {}
let errors: Record<string, string> = {}
let promises: Promise<void>[] = []
for (const sheet of sheets) {
const f = async (sheet: GoogleSpreadsheetWorksheet) => {
await utils.parallelForEach(
sheets,
async sheet => {
try {
await sheet.getRows({ limit: 1 })
} catch (err) {
@ -353,7 +353,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
if (
err.message.startsWith("No values in the header row") ||
err.message.startsWith("Header values are not yet loaded")
err.message.startsWith("Header values are not yet loaded") ||
err.message.startsWith("All your header cells are blank")
) {
errors[
sheet.title
@ -365,11 +366,9 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
// quietly.
throw err
}
}
promises.push(f(sheet))
}
await Promise.all(promises)
},
{ maxConcurrency: 2 }
)
for (const sheet of sheets) {
const id = buildExternalTableId(datasourceId, sheet.title)

83
packages/server/src/integrations/tests/googlesheets.spec.ts
View File

@ -506,8 +506,8 @@ describe("Google Sheets Integration", () => {
})
})
describe("import spreadsheet", () => {
it.only("should fail to import a completely blank sheet", async () => {
describe("fetch schema", () => {
it("should fail to import a completely blank sheet", async () => {
mock.createSheet({ title: "Sheet1" })
await config.api.datasource.fetchSchema(
{
@ -515,12 +515,87 @@ describe("Google Sheets Integration", () => {
tablesFilter: ["Sheet1"],
},
{
status: 400,
status: 200,
body: {
message: "",
errors: {
Sheet1:
'Failed to find a header row in sheet "Sheet1", is the first row blank?',
},
},
}
)
})
it("should fail to import multiple sheets with blank headers", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2"],
},
{
status: 200,
body: {
errors: {
Sheet1:
'Failed to find a header row in sheet "Sheet1", is the first row blank?',
Sheet2:
'Failed to find a header row in sheet "Sheet2", is the first row blank?',
},
},
}
)
})
it("should only fail the sheet with missing headers", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
mock.createSheet({ title: "Sheet3" })
mock.set("Sheet1!A1", "name")
mock.set("Sheet1!B1", "dob")
mock.set("Sheet2!A1", "name")
mock.set("Sheet2!B1", "dob")
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2", "Sheet3"],
},
{
status: 200,
body: {
errors: {
Sheet3:
'Failed to find a header row in sheet "Sheet3", is the first row blank?',
},
},
}
)
})
it("should only succeed if sheet with missing headers is not being imported", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
mock.createSheet({ title: "Sheet3" })
mock.set("Sheet1!A1", "name")
mock.set("Sheet1!B1", "dob")
mock.set("Sheet2!A1", "name")
mock.set("Sheet2!B1", "dob")
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2"],
},
{
status: 200,
body: { errors: {} },
}
)
})
})
})

40
packages/shared-core/src/utils.ts
View File

@ -12,33 +12,29 @@ interface PromiseWithId<T> {
id: number
}
export async function parallelForeach<T>(
export async function parallelForEach<T>(
items: T[],
task: (item: T) => Promise<void>,
maxConcurrency: number
opts?: { maxConcurrency?: number }
): Promise<void> {
try {
let next = 0
let inProgress: PromiseWithId<number>[] = []
while (next < items.length) {
if (inProgress.length === maxConcurrency) {
const finished = await Promise.race(inProgress.map(t => t.promise))
inProgress = inProgress.filter(task => task.id !== finished)
}
const promise = async (next: number) => {
await task(items[next])
return next
}
inProgress.push({ promise: promise(next), id: next })
next++
const { maxConcurrency = 10 } = opts || {}
let next = 0
let inProgress: PromiseWithId<number>[] = []
while (next < items.length) {
if (inProgress.length === maxConcurrency) {
const finished = await Promise.race(inProgress.map(t => t.promise))
inProgress = inProgress.filter(task => task.id !== finished)
}
await Promise.all(inProgress.map(t => t.promise))
} catch (e) {
console.error(e)
throw e
const promise = async (next: number) => {
await task(items[next])
return next
}
inProgress.push({ promise: promise(next), id: next })
next++
}
await Promise.all(inProgress.map(t => t.promise))
}
export function filterValueToLabel() {

85
yarn.lock
View File

@ -2053,6 +2053,44 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.6":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
"@budibase/pouchdb-replication-stream" "1.2.11"
"@budibase/shared-core" "0.0.0"
"@budibase/types" "0.0.0"
aws-cloudfront-sign "3.0.2"
aws-sdk "2.1030.0"
bcrypt "5.1.0"
bcryptjs "2.4.3"
bull "4.10.1"
correlation-id "4.0.0"
dd-trace "5.2.0"
dotenv "16.0.1"
ioredis "5.3.2"
joi "17.6.0"
jsonwebtoken "9.0.2"
knex "2.4.2"
koa-passport "^6.0.0"
koa-pino-logger "4.0.0"
lodash "4.17.21"
node-fetch "2.6.7"
passport-google-oauth "2.0.0"
passport-local "1.0.0"
passport-oauth2-refresh "^2.1.0"
pino "8.11.0"
pino-http "8.3.3"
posthog-node "4.0.1"
pouchdb "7.3.0"
pouchdb-find "7.2.2"
redlock "4.2.0"
rotating-file-stream "3.1.0"
sanitize-s3-objectkey "0.0.1"
semver "^7.5.4"
tar-fs "2.1.1"
uuid "^8.3.2"
"@budibase/handlebars-helpers@^0.13.2":
version "0.13.2"
resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@ -2095,6 +2133,45 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.32.6"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f"
integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ==
dependencies:
"@budibase/backend-core" "2.32.6"
"@budibase/shared-core" "2.32.6"
"@budibase/string-templates" "2.32.6"
"@budibase/types" "2.32.6"
"@koa/router" "8.0.8"
bull "4.10.1"
dd-trace "5.2.0"
joi "17.6.0"
jsonwebtoken "9.0.2"
lru-cache "^7.14.1"
memorystream "^0.3.1"
node-fetch "2.6.7"
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.6":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.32.6":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
dayjs "^1.10.8"
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.6":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"
"@bull-board/api@5.10.2":
version "5.10.2"
resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@ -12277,10 +12354,10 @@ google-p12-pem@^4.0.0:
dependencies:
node-forge "^1.3.1"
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5":
version "4.1.5"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f"
integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ==
dependencies:
axios "^1.4.0"
lodash "^4.17.21"