MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8270/validation-for-search-api 

# Conflicts:
#	packages/server/package.json
#	packages/server/src/api/controllers/row/index.ts
#	packages/server/src/api/routes/tests/search.spec.ts
#	yarn.lock
This commit is contained in:
Adria Navarro 2024-11-20 18:11:49 +01:00
parent 9da31211bb a031e16c56
commit 9c460424a9
27 changed files with 16550 additions and 15825 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/budibase_ci.yml vendored
View File

@ -114,9 +114,11 @@ jobs:
- name: Test - name: Test
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
else else
yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --no-prefix -- --verbose --reporters=default --reporters=github-actions yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --no-prefix
fi fi
test-worker: test-worker:

2
lerna.json
View File

@ -1,6 +1,6 @@
{ {
"$schema": "node_modules/lerna/schemas/lerna-schema.json", "$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.9", "version": "3.2.10",
"npmClient": "yarn", "npmClient": "yarn",
"concurrency": 20, "concurrency": 20,
"command": { "command": {

5
package.json
View File

@ -109,7 +109,7 @@
"semver": "7.5.3", "semver": "7.5.3",
"http-cache-semantics": "4.1.1", "http-cache-semantics": "4.1.1",
"msgpackr": "1.10.1", "msgpackr": "1.10.1",
"axios": "1.6.3", "axios": "1.7.7",
"xml2js": "0.6.2", "xml2js": "0.6.2",
"unset-value": "2.0.1", "unset-value": "2.0.1",
"passport": "0.6.0", "passport": "0.6.0",
@ -119,6 +119,5 @@
}, },
"engines": { "engines": {
"node": ">=20.0.0 <21.0.0" "node": ">=20.0.0 <21.0.0"
}, }
"dependencies": {}
} }

18
packages/backend-core/package.json
View File

@ -33,14 +33,17 @@
"@budibase/pouchdb-replication-stream": "1.2.11", "@budibase/pouchdb-replication-stream": "1.2.11",
"@budibase/shared-core": "0.0.0", "@budibase/shared-core": "0.0.0",
"@budibase/types": "0.0.0", "@budibase/types": "0.0.0",
"@techpass/passport-openidconnect": "0.3.3",
"aws-cloudfront-sign": "3.0.2", "aws-cloudfront-sign": "3.0.2",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"correlation-id": "4.0.0", "correlation-id": "4.0.0",
"dd-trace": "5.2.0", "dd-trace": "5.23.0",
"dotenv": "16.0.1", "dotenv": "16.0.1",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"joi": "17.6.0", "joi": "17.6.0",
"jsonwebtoken": "9.0.2", "jsonwebtoken": "9.0.2",
@ -55,17 +58,14 @@
"pino": "8.11.0", "pino": "8.11.0",
"pino-http": "8.3.3", "pino-http": "8.3.3",
"posthog-node": "4.0.1", "posthog-node": "4.0.1",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redlock": "4.2.0", "redlock": "4.2.0",
"rotating-file-stream": "3.1.0", "rotating-file-stream": "3.1.0",
"sanitize-s3-objectkey": "0.0.1", "sanitize-s3-objectkey": "0.0.1",
"semver": "^7.5.4", "semver": "^7.5.4",
"tar-fs": "2.1.1", "tar-fs": "2.1.1",
"uuid": "^8.3.2", "uuid": "^8.3.2"
"@techpass/passport-openidconnect": "0.3.3",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5"
}, },
"devDependencies": { "devDependencies": {
"@jest/types": "^29.6.3", "@jest/types": "^29.6.3",
@ -78,7 +78,7 @@
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/node": "^22.9.0", "@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/pouchdb": "6.4.0", "@types/pouchdb": "6.4.2",
"@types/redlock": "4.0.7", "@types/redlock": "4.0.7",
"@types/semver": "7.3.7", "@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1", "@types/tar-fs": "2.0.1",

2
packages/pro

@ -1 +1 @@
Subproject commit bfeece324a03a3a5f25137bf3f8c66d5ed6103d8 Subproject commit 4facf6a44ee52a405794845f71584168b9db652c

25
packages/server/package.json
View File

@ -63,13 +63,13 @@
"@bull-board/koa": "5.10.2", "@bull-board/koa": "5.10.2",
"@elastic/elasticsearch": "7.10.0", "@elastic/elasticsearch": "7.10.0",
"@google-cloud/firestore": "7.8.0", "@google-cloud/firestore": "7.8.0",
"@koa/router": "8.0.8", "@koa/router": "13.1.0",
"@socket.io/redis-adapter": "^8.2.1", "@socket.io/redis-adapter": "^8.2.1",
"@types/xml2js": "^0.4.14", "@types/xml2js": "^0.4.14",
"airtable": "0.12.2", "airtable": "0.12.2",
"arangojs": "7.2.0", "arangojs": "7.2.0",
"archiver": "7.0.1", "archiver": "7.0.1",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bson": "^6.9.0", "bson": "^6.9.0",
@ -80,8 +80,8 @@
"cookies": "0.8.0", "cookies": "0.8.0",
"csvtojson": "2.0.10", "csvtojson": "2.0.10",
"curlconverter": "3.21.0", "curlconverter": "3.21.0",
"dd-trace": "5.23.0",
"dayjs": "^1.10.8", "dayjs": "^1.10.8",
"dd-trace": "5.2.0",
"dotenv": "8.2.0", "dotenv": "8.2.0",
"form-data": "4.0.0", "form-data": "4.0.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",
@ -89,7 +89,7 @@
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"isolated-vm": "^4.7.2", "isolated-vm": "^4.7.2",
"jimp": "0.22.12", "jimp": "1.1.4",
"joi": "17.6.0", "joi": "17.6.0",
"js-yaml": "4.1.0", "js-yaml": "4.1.0",
"jsonschema": "1.4.0", "jsonschema": "1.4.0",
@ -104,7 +104,7 @@
"lodash": "4.17.21", "lodash": "4.17.21",
"memorystream": "0.3.1", "memorystream": "0.3.1",
"mongodb": "6.7.0", "mongodb": "6.7.0",
"mssql": "10.0.1", "mssql": "11.0.1",
"mysql2": "3.9.8", "mysql2": "3.9.8",
"node-fetch": "2.6.7", "node-fetch": "2.6.7",
"object-sizeof": "2.6.1", "object-sizeof": "2.6.1",
@ -112,15 +112,15 @@
"openapi-types": "9.3.1", "openapi-types": "9.3.1",
"oracledb": "6.5.1", "oracledb": "6.5.1",
"pg": "8.10.0", "pg": "8.10.0",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-all-dbs": "1.1.1", "pouchdb-all-dbs": "1.1.1",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redis": "4", "redis": "4",
"semver": "^7.5.4", "semver": "^7.5.4",
"serialize-error": "^7.0.1", "serialize-error": "^7.0.1",
"server-destroy": "1.0.1", "server-destroy": "1.0.1",
"snowflake-promise": "^4.5.0", "snowflake-sdk": "^1.15.0",
"socket.io": "4.7.5", "socket.io": "4.8.1",
"svelte": "^4.2.10", "svelte": "^4.2.10",
"tar": "6.2.1", "tar": "6.2.1",
"tmp": "0.2.3", "tmp": "0.2.3",
@ -128,7 +128,7 @@
"uuid": "^8.3.2", "uuid": "^8.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",
"worker-farm": "1.7.0", "worker-farm": "1.7.0",
"xml2js": "0.5.0", "xml2js": "0.6.2",
"zod-validation-error": "^3.4.0" "zod-validation-error": "^3.4.0"
}, },
"devDependencies": { "devDependencies": {
@ -142,13 +142,14 @@
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/koa": "2.13.4", "@types/koa": "2.13.4",
"@types/koa-send": "^4.1.6", "@types/koa-send": "^4.1.6",
"@types/koa__router": "8.0.8", "@types/koa__router": "12.0.4",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/mssql": "9.1.4", "@types/mssql": "9.1.5",
"@types/node": "^22.9.0", "@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/oracledb": "6.5.1", "@types/oracledb": "6.5.1",
"@types/pg": "8.6.6", "@types/pg": "8.6.6",
"@types/pouchdb": "6.4.2",
"@types/server-destroy": "1.0.1", "@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.14", "@types/supertest": "2.0.14",
"@types/tar": "6.1.5", "@types/tar": "6.1.5",

5
packages/server/src/api/controllers/query/import/sources/curl.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url"
const curlconverter = require("curlconverter") const curlconverter = require("curlconverter")
const parseCurl = (data: string): any => { const parseCurl = (data: string): Promise<any> => {
const curlJson = curlconverter.toJsonString(data) const curlJson = curlconverter.toJsonString(data)
return JSON.parse(curlJson) return JSON.parse(curlJson)
} }
@ -53,8 +53,7 @@ export class Curl extends ImportSource {
isSupported = async (data: string): Promise<boolean> => { isSupported = async (data: string): Promise<boolean> => {
try { try {
const curl = parseCurl(data) this.curl = parseCurl(data)
this.curl = curl
} catch (err) { } catch (err) {
return false return false
} }

13
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -164,9 +164,12 @@ describe("/datasources", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({
{ name: "%s", exclude: [DatabaseName.MONGODB, DatabaseName.SQS] }, exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
({ config, dsProvider }) => { })
if (descriptions.length) {
describe.each(descriptions)("$dbName", ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let rawDatasource: Datasource let rawDatasource: Datasource
let client: Knex let client: Knex
@ -492,5 +495,5 @@ datasourceDescribe(
) )
}) })
}) })
} })
) }

12
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -14,8 +14,13 @@ import { events } from "@budibase/backend-core"
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
datasourceDescribe( const descriptions = datasourceDescribe({
{ name: "queries (%s)", exclude: [DatabaseName.MONGODB, DatabaseName.SQS] }, exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
})
if (descriptions.length) {
describe.each(descriptions)(
"queries ($dbName)",
({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => { ({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => {
let rawDatasource: Datasource let rawDatasource: Datasource
let datasource: Datasource let datasource: Datasource
@ -945,4 +950,5 @@ datasourceDescribe(
}) })
}) })
} }
) )
}

10
packages/server/src/api/routes/tests/queries/mongodb.spec.ts
View File

@ -9,8 +9,11 @@ import { generator } from "@budibase/backend-core/tests"
const expectValidId = expect.stringMatching(/^\w{24}$/) const expectValidId = expect.stringMatching(/^\w{24}$/)
const expectValidBsonObjectId = expect.any(BSON.ObjectId) const expectValidBsonObjectId = expect.any(BSON.ObjectId)
datasourceDescribe( const descriptions = datasourceDescribe({ only: [DatabaseName.MONGODB] })
{ name: "/queries", only: [DatabaseName.MONGODB] },
if (descriptions.length) {
describe.each(descriptions)(
"/queries ($dbName)",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let collection: string let collection: string
let datasource: Datasource let datasource: Datasource
@ -714,4 +717,5 @@ datasourceDescribe(
}) })
}) })
} }
) )
}

35
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -85,8 +85,11 @@ function encodeJS(binding: string) {
return `{{ js "${Buffer.from(binding).toString("base64")}"}}` return `{{ js "${Buffer.from(binding).toString("base64")}"}}`
} }
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{ name: "/rows (%s)", exclude: [DatabaseName.MONGODB] },
if (descriptions.length) {
describe.each(descriptions)(
"/rows ($dbName)",
({ config, dsProvider, isInternal, isMSSQL, isOracle }) => { ({ config, dsProvider, isInternal, isMSSQL, isOracle }) => {
let table: Table let table: Table
let datasource: Datasource | undefined let datasource: Datasource | undefined
@ -338,7 +341,9 @@ datasourceDescribe(
await new Promise(r => setTimeout(r, Math.random() * 50)) await new Promise(r => setTimeout(r, Math.random() * 50))
} }
} }
throw new Error(`Failed to create row after ${attempts} attempts`) throw new Error(
`Failed to create row after ${attempts} attempts`
)
}) })
) )
@ -1495,7 +1500,9 @@ datasourceDescribe(
it("should return no errors on valid row", async () => { it("should return no errors on valid row", async () => {
const rowUsage = await getRowUsage() const rowUsage = await getRowUsage()
const res = await config.api.row.validate(table._id!, { name: "ivan" }) const res = await config.api.row.validate(table._id!, {
name: "ivan",
})
expect(res.valid).toBe(true) expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([]) expect(Object.keys(res.errors)).toEqual([])
@ -2244,7 +2251,10 @@ datasourceDescribe(
const table = await config.api.table.save(tableRequest) const table = await config.api.table.save(tableRequest)
const toCreate = generator const toCreate = generator
.unique(() => generator.integer({ min: 0, max: 10000 }), 10) .unique(() => generator.integer({ min: 0, max: 10000 }), 10)
.map(number => ({ number, string: generator.word({ length: 30 }) })) .map(number => ({
number,
string: generator.word({ length: 30 }),
}))
const rows = await Promise.all( const rows = await Promise.all(
toCreate.map(d => config.api.row.save(table._id!, d)) toCreate.map(d => config.api.row.save(table._id!, d))
@ -3019,7 +3029,10 @@ datasourceDescribe(
}, },
], ],
["from original saved row", (row: Row) => row], ["from original saved row", (row: Row) => row],
["from updated row", (row: Row) => config.api.row.save(viewId, row)], [
"from updated row",
(row: Row) => config.api.row.save(viewId, row),
],
] ]
it.each(testScenarios)( it.each(testScenarios)(
@ -3243,7 +3256,10 @@ datasourceDescribe(
async function updateFormulaColumn( async function updateFormulaColumn(
formula: string, formula: string,
opts?: { responseType?: FormulaResponseType; formulaType?: FormulaType } opts?: {
responseType?: FormulaResponseType
formulaType?: FormulaType
}
) { ) {
table = await config.api.table.save({ table = await config.api.table.save({
...table, ...table,
@ -3480,6 +3496,5 @@ datasourceDescribe(
}) })
}) })
} }
) )
}
// todo: remove me

12
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -977,8 +977,13 @@ describe("/rowsActions", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({
{ name: "row actions (%s)", only: [DatabaseName.SQS, DatabaseName.POSTGRES] }, only: [DatabaseName.SQS, DatabaseName.POSTGRES],
})
if (descriptions.length) {
describe.each(descriptions)(
"row actions ($dbName)",
({ config, dsProvider, isInternal }) => { ({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined let datasource: Datasource | undefined
@ -1036,4 +1041,5 @@ datasourceDescribe(
expect(await getRowActionsFromDb(tableId)).toBeUndefined() expect(await getRowActionsFromDb(tableId)).toBeUndefined()
}) })
} }
) )
}

395
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -60,11 +60,11 @@ jest.mock("@budibase/pro", () => ({
}, },
})) }))
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{
name: "search (%s)", if (descriptions.length) {
exclude: [DatabaseName.MONGODB], describe.each(descriptions)(
}, "search ($dbName)",
({ config, dsProvider, isInternal, isOracle, isSql }) => { ({ config, dsProvider, isInternal, isOracle, isSql }) => {
let datasource: Datasource | undefined let datasource: Datasource | undefined
let client: Knex | undefined let client: Knex | undefined
@ -199,7 +199,9 @@ datasourceDescribe(
]) ])
} }
describe.each(tableOrView)("from %s", (sourceType, createTableOrView) => { describe.each(tableOrView)(
"from %s",
(sourceType, createTableOrView) => {
const isView = sourceType === "view" const isView = sourceType === "view"
class SearchAssertion { class SearchAssertion {
@ -207,23 +209,9 @@ datasourceDescribe(
private async performSearch(): Promise<SearchResponse<Row>> { private async performSearch(): Promise<SearchResponse<Row>> {
if (isInMemory) { if (isInMemory) {
const query: RequiredKeys<Omit<RowSearchParams, "tableId">> = { return dataFilters.search(_.cloneDeep(rows), {
sort: this.query.sort, ...this.query,
query: this.query.query || {}, })
paginate: this.query.paginate,
bookmark: this.query.bookmark ?? undefined,
limit: this.query.limit,
sortOrder: this.query.sortOrder,
sortType: this.query.sortType,
version: this.query.version,
disableEscaping: this.query.disableEscaping,
countRows: this.query.countRows,
viewId: undefined,
fields: undefined,
indexer: undefined,
rows: undefined,
}
return dataFilters.search(_.cloneDeep(rows), query)
} else { } else {
return config.api.row.search(tableOrViewId, this.query) return config.api.row.search(tableOrViewId, this.query)
} }
@ -278,12 +266,16 @@ datasourceDescribe(
expectedRow: T, expectedRow: T,
foundRows: T[] foundRows: T[]
): NonNullable<T> { ): NonNullable<T> {
const row = foundRows.find(row => this.isMatch(expectedRow, row)) const row = foundRows.find(row =>
this.isMatch(expectedRow, row)
)
if (!row) { if (!row) {
const fields = Object.keys(expectedRow) const fields = Object.keys(expectedRow)
// To make the error message more readable, we only include the fields // To make the error message more readable, we only include the fields
// that are present in the expected row. // that are present in the expected row.
const searchedObjects = foundRows.map(row => _.pick(row, fields)) const searchedObjects = foundRows.map(row =>
_.pick(row, fields)
)
throw new Error( throw new Error(
`Failed to find row:\n\n${JSON.stringify( `Failed to find row:\n\n${JSON.stringify(
expectedRow, expectedRow,
@ -331,7 +323,9 @@ datasourceDescribe(
expect([...foundRows]).toEqual( expect([...foundRows]).toEqual(
expect.arrayContaining( expect.arrayContaining(
expectedRows.map((expectedRow: any) => expectedRows.map((expectedRow: any) =>
expect.objectContaining(this.popRow(expectedRow, foundRows)) expect.objectContaining(
this.popRow(expectedRow, foundRows)
)
) )
) )
) )
@ -359,7 +353,9 @@ datasourceDescribe(
} }
// Asserts that the query doesn't return a property, e.g. pagination parameters. // Asserts that the query doesn't return a property, e.g. pagination parameters.
async toNotHaveProperty(properties: (keyof SearchResponse<Row>)[]) { async toNotHaveProperty(
properties: (keyof SearchResponse<Row>)[]
) {
const response = await this.performSearch() const response = await this.performSearch()
const cloned = cloneDeep(response) const cloned = cloneDeep(response)
for (let property of properties) { for (let property of properties) {
@ -381,7 +377,9 @@ datasourceDescribe(
expect([...foundRows]).toEqual( expect([...foundRows]).toEqual(
expect.arrayContaining( expect.arrayContaining(
expectedRows.map((expectedRow: any) => expectedRows.map((expectedRow: any) =>
expect.objectContaining(this.popRow(expectedRow, foundRows)) expect.objectContaining(
this.popRow(expectedRow, foundRows)
)
) )
) )
) )
@ -418,15 +416,15 @@ datasourceDescribe(
describe("equal", () => { describe("equal", () => {
it("successfully finds true row", async () => { it("successfully finds true row", async () => {
await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ await expectQuery({ equal: { isTrue: true } }).toMatchExactly(
{ isTrue: true }, [{ isTrue: true }]
]) )
}) })
it("successfully finds false row", async () => { it("successfully finds false row", async () => {
await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ await expectQuery({
{ isTrue: false }, equal: { isTrue: false },
]) }).toMatchExactly([{ isTrue: false }])
}) })
}) })
@ -446,9 +444,9 @@ datasourceDescribe(
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds true row", async () => { it("successfully finds true row", async () => {
await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly( await expectQuery({
[{ isTrue: true }] oneOf: { isTrue: [true] },
) }).toContainExactly([{ isTrue: true }])
}) })
it("successfully finds false row", async () => { it("successfully finds false row", async () => {
@ -500,7 +498,10 @@ datasourceDescribe(
name: currentUser.firstName, name: currentUser.firstName,
appointment: future.toISOString(), appointment: future.toISOString(),
}, },
{ name: "serverDate", appointment: serverTime.toISOString() }, {
name: "serverDate",
appointment: serverTime.toISOString(),
},
{ {
name: "single user, session user", name: "single user, session user",
single_user: currentUser, single_user: currentUser,
@ -555,7 +556,10 @@ datasourceDescribe(
tableOrViewId = await createTableOrView({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
appointment: { name: "appointment", type: FieldType.DATETIME }, appointment: {
name: "appointment",
type: FieldType.DATETIME,
},
single_user: { single_user: {
name: "single_user", name: "single_user",
type: FieldType.BB_REFERENCE_SINGLE, type: FieldType.BB_REFERENCE_SINGLE,
@ -601,7 +605,9 @@ datasourceDescribe(
it("should return all rows matching the session user firstname when logical operator used", async () => { it("should return all rows matching the session user firstname when logical operator used", async () => {
await expectQuery({ await expectQuery({
$and: { $and: {
conditions: [{ equal: { name: "{{ [user].firstName }}" } }], conditions: [
{ equal: { name: "{{ [user].firstName }}" } },
],
}, },
}).toContainExactly([ }).toContainExactly([
{ {
@ -625,7 +631,10 @@ datasourceDescribe(
name: config.getUser().firstName, name: config.getUser().firstName,
appointment: future.toISOString(), appointment: future.toISOString(),
}, },
{ name: "serverDate", appointment: serverTime.toISOString() }, {
name: "serverDate",
appointment: serverTime.toISOString(),
},
]) ])
}) })
}) })
@ -641,7 +650,10 @@ datasourceDescribe(
}).toContainExactly([ }).toContainExactly([
{ name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
{ name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
{ name: "serverDate", appointment: serverTime.toISOString() }, {
name: "serverDate",
appointment: serverTime.toISOString(),
},
]) ])
}) })
@ -751,7 +763,9 @@ datasourceDescribe(
it("should not match the session user id in a deprecated multi user field", async () => { it("should not match the session user id in a deprecated multi user field", async () => {
await expectQuery({ await expectQuery({
notContains: { deprecated_multi_user: ["{{ [user]._id }}"] }, notContains: {
deprecated_multi_user: ["{{ [user]._id }}"],
},
notEmpty: { deprecated_multi_user: true }, notEmpty: { deprecated_multi_user: true },
}).toContainExactly([ }).toContainExactly([
{ {
@ -885,9 +899,9 @@ datasourceDescribe(
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { name: "foo" } }).toContainExactly([ await expectQuery({
{ name: "foo" }, equal: { name: "foo" },
]) }).toContainExactly([{ name: "foo" }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -912,27 +926,29 @@ datasourceDescribe(
describe("notEqual", () => { describe("notEqual", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ notEqual: { name: "foo" } }).toContainExactly( await expectQuery({
[{ name: "bar" }] notEqual: { name: "foo" },
) }).toContainExactly([{ name: "bar" }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { name: "bar" } }).toContainExactly( await expectQuery({
[{ name: "foo" }] notEqual: { name: "bar" },
) }).toContainExactly([{ name: "foo" }])
}) })
}) })
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ await expectQuery({
{ name: "foo" }, oneOf: { name: ["foo"] },
]) }).toContainExactly([{ name: "foo" }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() await expectQuery({
oneOf: { name: ["none"] },
}).toFindNothing()
}) })
it("can have multiple values for same column", async () => { it("can have multiple values for same column", async () => {
@ -980,9 +996,9 @@ datasourceDescribe(
describe("fuzzy", () => { describe("fuzzy", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly(
{ name: "foo" }, [{ name: "foo" }]
]) )
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -992,19 +1008,21 @@ datasourceDescribe(
describe("string", () => { describe("string", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ string: { name: "fo" } }).toContainExactly([ await expectQuery({
{ name: "foo" }, string: { name: "fo" },
]) }).toContainExactly([{ name: "foo" }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ string: { name: "none" } }).toFindNothing() await expectQuery({
string: { name: "none" },
}).toFindNothing()
}) })
it("is case-insensitive", async () => { it("is case-insensitive", async () => {
await expectQuery({ string: { name: "FO" } }).toContainExactly([ await expectQuery({
{ name: "foo" }, string: { name: "FO" },
]) }).toContainExactly([{ name: "foo" }])
}) })
}) })
@ -1063,10 +1081,9 @@ datasourceDescribe(
describe("notEmpty", () => { describe("notEmpty", () => {
it("finds all non-empty rows", async () => { it("finds all non-empty rows", async () => {
await expectQuery({ notEmpty: { name: null } }).toContainExactly([ await expectQuery({
{ name: "foo" }, notEmpty: { name: null },
{ name: "bar" }, }).toContainExactly([{ name: "foo" }, { name: "bar" }])
])
}) })
it("should not be affected by when filter empty behaviour", async () => { it("should not be affected by when filter empty behaviour", async () => {
@ -1182,9 +1199,9 @@ datasourceDescribe(
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ await expectQuery({ notEqual: { age: 10 } }).toContainExactly(
{ age: 1 }, [{ age: 1 }]
]) )
}) })
}) })
@ -1332,9 +1349,9 @@ datasourceDescribe(
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ await expectQuery({
{ dob: JAN_1ST }, equal: { dob: JAN_1ST },
]) }).toContainExactly([{ dob: JAN_1ST }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1358,13 +1375,15 @@ datasourceDescribe(
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly( await expectQuery({
[{ dob: JAN_1ST }] oneOf: { dob: [JAN_1ST] },
) }).toContainExactly([{ dob: JAN_1ST }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() await expectQuery({
oneOf: { dob: [JAN_2ND] },
}).toFindNothing()
}) })
}) })
@ -1396,7 +1415,10 @@ datasourceDescribe(
it("greater than equal to", async () => { it("greater than equal to", async () => {
await expectQuery({ await expectQuery({
range: { range: {
dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() }, dob: {
low: JAN_10TH,
high: MAX_VALID_DATE.toISOString(),
},
}, },
}).toContainExactly([{ dob: JAN_10TH }]) }).toContainExactly([{ dob: JAN_10TH }])
}) })
@ -1499,9 +1521,9 @@ datasourceDescribe(
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { time: T_1000 } }).toContainExactly( await expectQuery({
[{ time: "10:00:00" }] equal: { time: T_1000 },
) }).toContainExactly([{ time: "10:00:00" }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1707,7 +1729,9 @@ datasourceDescribe(
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ await expectQuery({
oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] }, oneOf: {
ai: ["Mock LLM Response", "Other LLM Response"],
},
}).toContainExactly([ }).toContainExactly([
{ product: "Big Mac" }, { product: "Big Mac" },
{ product: "McCrispy" }, { product: "McCrispy" },
@ -1760,9 +1784,12 @@ datasourceDescribe(
}) })
it("finds all with empty list", async () => { it("finds all with empty list", async () => {
await expectQuery({ contains: { numbers: [] } }).toContainExactly( await expectQuery({
[{ numbers: ["one", "two"] }, { numbers: ["three"] }] contains: { numbers: [] },
) }).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
])
}) })
}) })
@ -1832,14 +1859,18 @@ datasourceDescribe(
tableOrViewId = await createTableOrView({ tableOrViewId = await createTableOrView({
num: { name: "num", type: FieldType.BIGINT }, num: { name: "num", type: FieldType.BIGINT },
}) })
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) await createRows([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
])
}) })
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { num: SMALL } }).toContainExactly([ await expectQuery({ equal: { num: SMALL } }).toContainExactly(
{ num: SMALL }, [{ num: SMALL }]
]) )
}) })
it("successfully finds a big value", async () => { it("successfully finds a big value", async () => {
@ -1855,26 +1886,23 @@ datasourceDescribe(
describe("notEqual", () => { describe("notEqual", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ await expectQuery({
{ num: MEDIUM }, notEqual: { num: SMALL },
{ num: BIG }, }).toContainExactly([{ num: MEDIUM }, { num: BIG }])
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ await expectQuery({ notEqual: { num: 10 } }).toContainExactly(
{ num: SMALL }, [{ num: SMALL }, { num: MEDIUM }, { num: BIG }]
{ num: MEDIUM }, )
{ num: BIG },
])
}) })
}) })
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ await expectQuery({
{ num: SMALL }, oneOf: { num: [SMALL] },
]) }).toContainExactly([{ num: SMALL }])
}) })
it("successfully finds all rows", async () => { it("successfully finds all rows", async () => {
@ -1959,7 +1987,9 @@ datasourceDescribe(
describe("not equal", () => { describe("not equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ await expectQuery({
notEqual: { auto: 1 },
}).toContainExactly([
{ auto: 2 }, { auto: 2 },
{ auto: 3 }, { auto: 3 },
{ auto: 4 }, { auto: 4 },
@ -1973,7 +2003,9 @@ datasourceDescribe(
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ await expectQuery({
notEqual: { auto: 0 },
}).toContainExactly([
{ auto: 1 }, { auto: 1 },
{ auto: 2 }, { auto: 2 },
{ auto: 3 }, { auto: 3 },
@ -1990,9 +2022,9 @@ datasourceDescribe(
describe("oneOf", () => { describe("oneOf", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ await expectQuery({
{ auto: 1 }, oneOf: { auto: [1] },
]) }).toContainExactly([{ auto: 1 }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -2096,13 +2128,16 @@ datasourceDescribe(
hasNextPage: boolean | undefined = true, hasNextPage: boolean | undefined = true,
rowCount: number = 0 rowCount: number = 0
do { do {
const response = await config.api.row.search(tableOrViewId, { const response = await config.api.row.search(
tableOrViewId,
{
tableId: tableOrViewId, tableId: tableOrViewId,
limit: 1, limit: 1,
paginate: true, paginate: true,
query: {}, query: {},
bookmark, bookmark,
}) }
)
bookmark = response.bookmark bookmark = response.bookmark
hasNextPage = response.hasNextPage hasNextPage = response.hasNextPage
expect(response.rows.length).toEqual(1) expect(response.rows.length).toEqual(1)
@ -2120,13 +2155,16 @@ datasourceDescribe(
// eslint-disable-next-line no-constant-condition // eslint-disable-next-line no-constant-condition
while (true) { while (true) {
const response = await config.api.row.search(tableOrViewId, { const response = await config.api.row.search(
tableOrViewId,
{
tableId: tableOrViewId, tableId: tableOrViewId,
limit: 3, limit: 3,
query: {}, query: {},
bookmark, bookmark,
paginate: true, paginate: true,
}) }
)
rows.push(...response.rows) rows.push(...response.rows)
@ -2159,7 +2197,9 @@ datasourceDescribe(
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() await expectQuery({
equal: { "1:1:name": "none" },
}).toFindNothing()
}) })
}) })
@ -2236,7 +2276,11 @@ datasourceDescribe(
}, },
}) })
await createRows([{ user: user1 }, { user: user2 }, { user: null }]) await createRows([
{ user: user1 },
{ user: user2 },
{ user: null },
])
}) })
describe("equal", () => { describe("equal", () => {
@ -2247,7 +2291,9 @@ datasourceDescribe(
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ equal: { user: "us_none" } }).toFindNothing() await expectQuery({
equal: { user: "us_none" },
}).toFindNothing()
}) })
}) })
@ -2285,15 +2331,17 @@ datasourceDescribe(
describe("empty", () => { describe("empty", () => {
it("finds empty rows", async () => { it("finds empty rows", async () => {
await expectQuery({ empty: { user: null } }).toContainExactly([ await expectQuery({ empty: { user: null } }).toContainExactly(
{}, [{}]
]) )
}) })
}) })
describe("notEmpty", () => { describe("notEmpty", () => {
it("finds non-empty rows", async () => { it("finds non-empty rows", async () => {
await expectQuery({ notEmpty: { user: null } }).toContainExactly([ await expectQuery({
notEmpty: { user: null },
}).toContainExactly([
{ user: { _id: user1._id } }, { user: { _id: user1._id } },
{ user: { _id: user2._id } }, { user: { _id: user2._id } },
]) ])
@ -2400,7 +2448,9 @@ datasourceDescribe(
await expectQuery({ await expectQuery({
equal: { number: 1 }, equal: { number: 1 },
contains: { users: [user1._id] }, contains: { users: [user1._id] },
}).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) }).toContainExactly([
{ users: [{ _id: user1._id }], number: 1 },
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -2423,15 +2473,18 @@ datasourceDescribe(
let productCategoryTable: Table, productCatRows: Row[] let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => { beforeAll(async () => {
const { relatedTable, tableId } = await basicRelationshipTables( const { relatedTable, tableId } =
relationshipType await basicRelationshipTables(relationshipType)
)
tableOrViewId = tableId tableOrViewId = tableId
productCategoryTable = relatedTable productCategoryTable = relatedTable
productCatRows = await Promise.all([ productCatRows = await Promise.all([
config.api.row.save(productCategoryTable._id!, { name: "foo" }), config.api.row.save(productCategoryTable._id!, {
config.api.row.save(productCategoryTable._id!, { name: "bar" }), name: "foo",
}),
config.api.row.save(productCategoryTable._id!, {
name: "bar",
}),
]) ])
await Promise.all([ await Promise.all([
@ -2454,7 +2507,10 @@ datasourceDescribe(
await expectQuery({ await expectQuery({
equal: { ["productCat.name"]: "foo" }, equal: { ["productCat.name"]: "foo" },
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, {
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
]) ])
}) })
@ -2462,7 +2518,10 @@ datasourceDescribe(
await expectQuery({ await expectQuery({
equal: { [`${productCategoryTable.name}.name`]: "foo" }, equal: { [`${productCategoryTable.name}.name`]: "foo" },
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, {
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
]) ])
}) })
@ -2473,7 +2532,10 @@ datasourceDescribe(
}) })
describe("logical filters", () => { describe("logical filters", () => {
const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR] const logicalOperators = [
LogicalOperator.AND,
LogicalOperator.OR,
]
describe("$and", () => { describe("$and", () => {
it("should allow single conditions", async () => { it("should allow single conditions", async () => {
@ -2714,9 +2776,8 @@ datasourceDescribe(
RelationshipType.MANY_TO_MANY, RelationshipType.MANY_TO_MANY,
])("big relations (%s)", relationshipType => { ])("big relations (%s)", relationshipType => {
beforeAll(async () => { beforeAll(async () => {
const { relatedTable, tableId } = await basicRelationshipTables( const { relatedTable, tableId } =
relationshipType await basicRelationshipTables(relationshipType)
)
tableOrViewId = tableId tableOrViewId = tableId
const mainRow = await config.api.row.save(tableOrViewId, { const mainRow = await config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
@ -2730,12 +2791,15 @@ datasourceDescribe(
}) })
it("can only pull 10 related rows", async () => { it("can only pull 10 related rows", async () => {
await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { await withCoreEnv(
{ SQL_MAX_RELATED_ROWS: "10" },
async () => {
const response = await expectQuery({}).toContain([ const response = await expectQuery({}).toContain([
{ name: "foo" }, { name: "foo" },
]) ])
expect(response.rows[0].productCat).toBeArrayOfSize(10) expect(response.rows[0].productCat).toBeArrayOfSize(10)
}) }
)
}) })
it("can pull max rows when env not set (defaults to 500)", async () => { it("can pull max rows when env not set (defaults to 500)", async () => {
@ -2950,9 +3014,11 @@ datasourceDescribe(
}) })
}) })
describe.each(["data_name_test", "name_data_test", "name_test_data_"])( describe.each([
"special (%s) case", "data_name_test",
column => { "name_data_test",
"name_test_data_",
])("special (%s) case", column => {
beforeAll(async () => { beforeAll(async () => {
tableOrViewId = await createTableOrView({ tableOrViewId = await createTableOrView({
[column]: { [column]: {
@ -2972,8 +3038,7 @@ datasourceDescribe(
}, },
}).toContainExactly([{ [column]: "a" }]) }).toContainExactly([{ [column]: "a" }])
}) })
} })
)
isInternal && isInternal &&
describe("sample data", () => { describe("sample data", () => {
@ -2995,10 +3060,22 @@ datasourceDescribe(
}) })
describe.each([ describe.each([
{ low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, {
{ low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, low: "2024-07-03T00:00:00.000Z",
{ low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, high: "9999-00-00T00:00:00.000Z",
{ low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, },
{
low: "2024-07-03T00:00:00.000Z",
high: "9998-00-00T00:00:00.000Z",
},
{
low: "0000-00-00T00:00:00.000Z",
high: "2024-07-04T00:00:00.000Z",
},
{
low: "0001-00-00T00:00:00.000Z",
high: "2024-07-04T00:00:00.000Z",
},
])("date special cases", ({ low, high }) => { ])("date special cases", ({ low, high }) => {
const earlyDate = "2024-07-03T10:00:00.000Z", const earlyDate = "2024-07-03T10:00:00.000Z",
laterDate = "2024-07-03T11:00:00.000Z" laterDate = "2024-07-03T11:00:00.000Z"
@ -3275,13 +3352,17 @@ datasourceDescribe(
}, },
}) })
const toRelateTable = await config.api.table.get(toRelateTableId) const toRelateTable = await config.api.table.get(
toRelateTableId
)
await config.api.table.save({ await config.api.table.save({
...toRelateTable, ...toRelateTable,
primaryDisplay: "link", primaryDisplay: "link",
}) })
const relatedRows = await Promise.all([ const relatedRows = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "related" }), config.api.row.save(toRelateTable._id!, {
name: "related",
}),
]) ])
await config.api.row.save(tableOrViewId, { await config.api.row.save(tableOrViewId, {
name: "test", name: "test",
@ -3660,7 +3741,9 @@ datasourceDescribe(
"'; SHUTDOWN --", "'; SHUTDOWN --",
] ]
describe.each(badStrings)("bad string: %s", badStringTemplate => { describe.each(badStrings)(
"bad string: %s",
badStringTemplate => {
// The SQL that knex generates when you try to use a double quote in a // The SQL that knex generates when you try to use a double quote in a
// field name is always invalid and never works, so we skip it for these // field name is always invalid and never works, so we skip it for these
// tests. // tests.
@ -3680,12 +3763,17 @@ datasourceDescribe(
...table, ...table,
schema: { schema: {
...table.schema, ...table.schema,
[badString]: { name: badString, type: FieldType.STRING }, [badString]: {
name: badString,
type: FieldType.STRING,
},
}, },
}) })
if (docIds.isViewId(tableOrViewId)) { if (docIds.isViewId(tableOrViewId)) {
const view = await config.api.viewV2.get(tableOrViewId) const view = await config.api.viewV2.get(
tableOrViewId
)
await config.api.viewV2.update({ await config.api.viewV2.update({
...view, ...view,
schema: { schema: {
@ -3741,9 +3829,12 @@ datasourceDescribe(
await assertTableExists(table) await assertTableExists(table)
await assertTableNumRows(table, 1) await assertTableNumRows(table, 1)
}) })
}) }
}) )
})
}) })
} }
) )
})
}
)
}

40
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -38,8 +38,11 @@ import timekeeper from "timekeeper"
const { basicTable } = setup.structures const { basicTable } = setup.structures
const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{ name: "/tables (%s)", exclude: [DatabaseName.MONGODB] },
if (descriptions.length) {
describe.each(descriptions)(
"/tables ($dbName)",
({ config, dsProvider, isInternal, isOracle }) => { ({ config, dsProvider, isInternal, isOracle }) => {
let datasource: Datasource | undefined let datasource: Datasource | undefined
@ -332,7 +335,9 @@ datasourceDescribe(
expect(updatedTable).toEqual(expect.objectContaining(expected)) expect(updatedTable).toEqual(expect.objectContaining(expected))
const persistedTable = await config.api.table.get(updatedTable._id!) const persistedTable = await config.api.table.get(
updatedTable._id!
)
expected = { expected = {
...table, ...table,
name: newName, name: newName,
@ -561,8 +566,14 @@ datasourceDescribe(
await config.api.table.save(saveTableRequest, { await config.api.table.save(saveTableRequest, {
status: 200, status: 200,
}) })
saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" } saveTableRequest.schema.foo = {
saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" } type: FieldType.STRING,
name: "foo",
}
saveTableRequest.schema.FOO = {
type: FieldType.STRING,
name: "FOO",
}
await config.api.table.save(saveTableRequest, { await config.api.table.save(saveTableRequest, {
status: 400, status: 400,
@ -1180,10 +1191,12 @@ datasourceDescribe(
schema, schema,
}) })
) )
const result = await config.api.table.validateExistingTableImport({ const result = await config.api.table.validateExistingTableImport(
{
tableId: table._id, tableId: table._id,
rows, rows,
}) }
)
return result return result
}, },
], ],
@ -1267,7 +1280,9 @@ datasourceDescribe(
isInternal && isInternal &&
it.each( it.each(
isInternal ? PROTECTED_INTERNAL_COLUMNS : PROTECTED_EXTERNAL_COLUMNS isInternal
? PROTECTED_INTERNAL_COLUMNS
: PROTECTED_EXTERNAL_COLUMNS
)( )(
"don't allow protected names in the rows (%s)", "don't allow protected names in the rows (%s)",
async columnName => { async columnName => {
@ -1487,7 +1502,8 @@ datasourceDescribe(
schema: basicSchema, schema: basicSchema,
}) })
) )
const result = await config.api.table.validateExistingTableImport({ const result = await config.api.table.validateExistingTableImport(
{
tableId: table._id, tableId: table._id,
rows: [ rows: [
{ {
@ -1496,7 +1512,8 @@ datasourceDescribe(
name: generator.first(), name: generator.first(),
}, },
], ],
}) }
)
expect(result).toEqual({ expect(result).toEqual({
allValid: true, allValid: true,
@ -1512,4 +1529,5 @@ datasourceDescribe(
}) })
}) })
} }
) )
}

50
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -44,8 +44,11 @@ import merge from "lodash/merge"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { db, roles, context } from "@budibase/backend-core" import { db, roles, context } from "@budibase/backend-core"
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{ name: "/v2/views (%s)", exclude: [DatabaseName.MONGODB] },
if (descriptions.length) {
describe.each(descriptions)(
"/v2/views ($dbName)",
({ config, isInternal, dsProvider }) => { ({ config, isInternal, dsProvider }) => {
let table: Table let table: Table
let rawDatasource: Datasource | undefined let rawDatasource: Datasource | undefined
@ -129,7 +132,8 @@ datasourceDescribe(
}) })
it("can persist views with all fields", async () => { it("can persist views with all fields", async () => {
const newView: Required<Omit<CreateViewRequest, "query" | "type">> = { const newView: Required<Omit<CreateViewRequest, "query" | "type">> =
{
name: generator.name(), name: generator.name(),
tableId: table._id!, tableId: table._id!,
primaryDisplay: "id", primaryDisplay: "id",
@ -194,8 +198,9 @@ datasourceDescribe(
}) })
it("can create a view with just a query field, no queryUI, for backwards compatibility", async () => { it("can create a view with just a query field, no queryUI, for backwards compatibility", async () => {
const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = const newView: Required<
{ Omit<CreateViewRequest, "queryUI" | "type">
> = {
name: generator.name(), name: generator.name(),
tableId: table._id!, tableId: table._id!,
primaryDisplay: "id", primaryDisplay: "id",
@ -1162,7 +1167,8 @@ datasourceDescribe(
.expect(400) .expect(400)
expect(result.body).toEqual({ expect(result.body).toEqual({
message: "View id does not match between the body and the uri path", message:
"View id does not match between the body and the uri path",
status: 400, status: 400,
}) })
}) })
@ -2016,7 +2022,10 @@ datasourceDescribe(
schema, schema,
}) })
const renameColumn = async (table: Table, renaming: RenameColumn) => { const renameColumn = async (
table: Table,
renaming: RenameColumn
) => {
const newSchema = { ...table.schema } const newSchema = { ...table.schema }
newSchema[renaming.updated] = { newSchema[renaming.updated] = {
...table.schema[renaming.old], ...table.schema[renaming.old],
@ -2617,7 +2626,9 @@ datasourceDescribe(
]) ])
const rowUsage = await getRowUsage() const rowUsage = await getRowUsage()
await config.api.row.bulkDelete(view.id, { rows: [rows[0], rows[2]] }) await config.api.row.bulkDelete(view.id, {
rows: [rows[0], rows[2]],
})
await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage) await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
@ -3470,7 +3481,10 @@ datasourceDescribe(
expect(response.rows).toEqual( expect(response.rows).toEqual(
expect.arrayContaining([ expect.arrayContaining([
expect.objectContaining({ expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), "Quantity Sum": rows.reduce(
(acc, r) => acc + r.quantity,
0
),
}), }),
]) ])
) )
@ -3511,7 +3525,9 @@ datasourceDescribe(
} }
for (const row of response.rows) { for (const row of response.rows) {
expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) expect(row["Total Price"]).toEqual(
priceByQuantity[row.quantity]
)
} }
}) })
@ -3701,9 +3717,12 @@ datasourceDescribe(
}, },
}) })
const apertureScience = await config.api.row.save(companies._id!, { const apertureScience = await config.api.row.save(
companies._id!,
{
name: "Aperture Science Laboratories", name: "Aperture Science Laboratories",
}) }
)
const blackMesa = await config.api.row.save(companies._id!, { const blackMesa = await config.api.row.save(companies._id!, {
name: "Black Mesa", name: "Black Mesa",
@ -4402,7 +4421,9 @@ datasourceDescribe(
}), }),
expected: () => [ expected: () => [
{ {
user: expect.objectContaining({ _id: config.getUser()._id }), user: expect.objectContaining({
_id: config.getUser()._id,
}),
}, },
], ],
}, },
@ -4631,4 +4652,5 @@ datasourceDescribe(
}) })
}) })
} }
) )
}

13
packages/server/src/automations/tests/executeQuery.spec.ts
View File

@ -7,11 +7,13 @@ import {
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
datasourceDescribe( const descriptions = datasourceDescribe({
{
name: "execute query action",
exclude: [DatabaseName.MONGODB, DatabaseName.SQS], exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
}, })
if (descriptions.length) {
describe.each(descriptions)(
"execute query action ($dbName)",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let tableName: string let tableName: string
let client: Knex let client: Knex
@ -74,4 +76,5 @@ datasourceDescribe(
expect(res.success).toEqual(false) expect(res.success).toEqual(false)
}) })
} }
) )
}

11
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -433,9 +433,10 @@ describe("Automation Scenarios", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
{ name: "", only: [DatabaseName.MYSQL] },
({ config, dsProvider }) => { if (descriptions.length) {
describe.each(descriptions)("/rows ($dbName)", ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -531,5 +532,5 @@ datasourceDescribe(
) )
}) })
}) })
} })
) }

30
packages/server/src/integration-test/mysql.spec.ts
View File

@ -10,11 +10,11 @@ function uniqueTableName(length?: number): string {
.substring(0, length || 10) .substring(0, length || 10)
} }
datasourceDescribe( const mainDescriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
{
name: "Integration compatibility with mysql search_path", if (mainDescriptions.length) {
only: [DatabaseName.MYSQL], describe.each(mainDescriptions)(
}, "/Integration compatibility with mysql search_path ($dbName)",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let rawDatasource: Datasource let rawDatasource: Datasource
let datasource: Datasource let datasource: Datasource
@ -71,18 +71,20 @@ datasourceDescribe(
datasourceId: datasource._id!, datasourceId: datasource._id!,
tablesFilter: [repeated_table_name], tablesFilter: [repeated_table_name],
}) })
expect(res.datasource.entities![repeated_table_name].schema).toBeDefined() expect(
res.datasource.entities![repeated_table_name].schema
).toBeDefined()
const schema = res.datasource.entities![repeated_table_name].schema const schema = res.datasource.entities![repeated_table_name].schema
expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
}) })
} }
) )
datasourceDescribe( const descriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
{
name: "POST /api/datasources/:datasourceId/schema", if (descriptions.length) {
only: [DatabaseName.MYSQL], describe.each(descriptions)(
}, "POST /api/datasources/:datasourceId/schema ($dbName)",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -125,4 +127,6 @@ datasourceDescribe(
expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
}) })
} }
) )
}
}

31
packages/server/src/integration-test/postgres.spec.ts
View File

@ -8,8 +8,11 @@ import {
} from "../integrations/tests/utils" } from "../integrations/tests/utils"
import { Knex } from "knex" import { Knex } from "knex"
datasourceDescribe( const mainDescriptions = datasourceDescribe({ only: [DatabaseName.POSTGRES] })
{ name: "postgres integrations", only: [DatabaseName.POSTGRES] },
if (mainDescriptions.length) {
describe.each(mainDescriptions)(
"/postgres integrations",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -199,18 +202,21 @@ datasourceDescribe(
row = await config.api.row.save(table._id!, { ...row, price: 300 }) row = await config.api.row.save(table._id!, { ...row, price: 300 })
expect(row.price).toBe("300.00") expect(row.price).toBe("300.00")
row = await config.api.row.save(table._id!, { ...row, price: "400.00" }) row = await config.api.row.save(table._id!, {
...row,
price: "400.00",
})
expect(row.price).toBe("400.00") expect(row.price).toBe("400.00")
}) })
}) })
} }
) )
datasourceDescribe( const descriptions = datasourceDescribe({ only: [DatabaseName.POSTGRES] })
{
name: "Integration compatibility with postgres search_path", if (descriptions.length) {
only: [DatabaseName.POSTGRES], describe.each(descriptions)(
}, "Integration compatibility with postgres search_path",
({ config, dsProvider }) => { ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -283,8 +289,11 @@ datasourceDescribe(
expect( expect(
response.datasource.entities?.[repeated_table_name].schema response.datasource.entities?.[repeated_table_name].schema
).toBeDefined() ).toBeDefined()
const schema = response.datasource.entities?.[repeated_table_name].schema const schema =
response.datasource.entities?.[repeated_table_name].schema
expect(Object.keys(schema || {}).sort()).toEqual(["id", "val1"]) expect(Object.keys(schema || {}).sort()).toEqual(["id", "val1"])
}) })
} }
) )
}
}

6
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -281,8 +281,14 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
case MSSQLConfigAuthType.NTLM: { case MSSQLConfigAuthType.NTLM: {
const { domain, trustServerCertificate } = const { domain, trustServerCertificate } =
this.config.ntlmConfig || {} this.config.ntlmConfig || {}
if (!domain) {
throw Error("Domain must be provided for NTLM config")
}
clientCfg.authentication = { clientCfg.authentication = {
type: "ntlm", type: "ntlm",
// @ts-expect-error - username and password not required for NTLM
options: { options: {
domain, domain,
}, },

50
packages/server/src/integrations/snowflake.ts
View File

@ -6,7 +6,8 @@ import {
QueryType, QueryType,
SqlQuery, SqlQuery,
} from "@budibase/types" } from "@budibase/types"
import { Snowflake } from "snowflake-promise" import snowflakeSdk, { SnowflakeError } from "snowflake-sdk"
import { promisify } from "util"
interface SnowflakeConfig { interface SnowflakeConfig {
account: string account: string
@ -71,11 +72,52 @@ const SCHEMA: Integration = {
}, },
} }
class SnowflakeIntegration { class SnowflakePromise {
private client: Snowflake config: SnowflakeConfig
client?: snowflakeSdk.Connection
constructor(config: SnowflakeConfig) { constructor(config: SnowflakeConfig) {
this.client = new Snowflake(config) this.config = config
}
async connect() {
if (this.client?.isUp()) return
this.client = snowflakeSdk.createConnection(this.config)
const connectAsync = promisify(this.client.connect.bind(this.client))
return connectAsync()
}
async execute(sql: string) {
return new Promise((resolve, reject) => {
if (!this.client) {
throw Error(
"No snowflake client present to execute query. Run connect() first to initialise."
)
}
this.client.execute({
sqlText: sql,
complete: function (
err: SnowflakeError | undefined,
statementExecuted: any,
rows: any
) {
if (err) {
return reject(err)
}
resolve(rows)
},
})
})
}
}
class SnowflakeIntegration {
private client: SnowflakePromise
constructor(config: SnowflakeConfig) {
this.client = new SnowflakePromise(config)
} }
async testConnection(): Promise<ConnectionInfo> { async testConnection(): Promise<ConnectionInfo> {

41
packages/server/src/integrations/tests/utils/index.ts
View File

@ -35,7 +35,6 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
} }
export interface DatasourceDescribeOpts { export interface DatasourceDescribeOpts {
name: string
only?: DatabaseName[] only?: DatabaseName[]
exclude?: DatabaseName[] exclude?: DatabaseName[]
} }
@ -102,16 +101,12 @@ function createDummyTest() {
}) })
} }
export function datasourceDescribe( export function datasourceDescribe(opts: DatasourceDescribeOpts) {
opts: DatasourceDescribeOpts,
cb: (args: DatasourceDescribeReturn) => void
) {
if (process.env.DATASOURCE === "none") { if (process.env.DATASOURCE === "none") {
createDummyTest() createDummyTest()
return
} }
const { name, only, exclude } = opts const { only, exclude } = opts
if (only && exclude) { if (only && exclude) {
throw new Error("you can only supply one of 'only' or 'exclude'") throw new Error("you can only supply one of 'only' or 'exclude'")
@ -130,36 +125,28 @@ export function datasourceDescribe(
if (databases.length === 0) { if (databases.length === 0) {
createDummyTest() createDummyTest()
return
} }
describe.each(databases)(name, name => {
const config = new TestConfiguration() const config = new TestConfiguration()
return databases.map(dbName => ({
afterAll(() => { dbName,
config.end()
})
cb({
name,
config, config,
dsProvider: () => createDatasources(config, name), dsProvider: () => createDatasources(config, dbName),
isInternal: name === DatabaseName.SQS, isInternal: dbName === DatabaseName.SQS,
isExternal: name !== DatabaseName.SQS, isExternal: dbName !== DatabaseName.SQS,
isSql: [ isSql: [
DatabaseName.MARIADB, DatabaseName.MARIADB,
DatabaseName.MYSQL, DatabaseName.MYSQL,
DatabaseName.POSTGRES, DatabaseName.POSTGRES,
DatabaseName.SQL_SERVER, DatabaseName.SQL_SERVER,
DatabaseName.ORACLE, DatabaseName.ORACLE,
].includes(name), ].includes(dbName),
isMySQL: name === DatabaseName.MYSQL, isMySQL: dbName === DatabaseName.MYSQL,
isPostgres: name === DatabaseName.POSTGRES, isPostgres: dbName === DatabaseName.POSTGRES,
isMongodb: name === DatabaseName.MONGODB, isMongodb: dbName === DatabaseName.MONGODB,
isMSSQL: name === DatabaseName.SQL_SERVER, isMSSQL: dbName === DatabaseName.SQL_SERVER,
isOracle: name === DatabaseName.ORACLE, isOracle: dbName === DatabaseName.ORACLE,
}) }))
})
} }
function getDatasource( function getDatasource(

10
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -19,8 +19,11 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures"
// These test cases are only for things that cannot be tested through the API // These test cases are only for things that cannot be tested through the API
// (e.g. limiting searches to returning specific fields). If it's possible to // (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead. // test through the API, it should be done there instead.
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{ name: "search sdk (%s)", exclude: [DatabaseName.MONGODB] },
if (descriptions.length) {
describe.each(descriptions)(
"search sdk ($dbName)",
({ config, dsProvider, isInternal }) => { ({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined let datasource: Datasource | undefined
let table: Table let table: Table
@ -217,4 +220,5 @@ datasourceDescribe(
} }
) )
} }
) )
}

6
packages/server/src/utilities/fileSystem/processor.ts
View File

@ -1,4 +1,4 @@
import jimp from "jimp" import { Jimp } from "jimp"
const FORMATS = { const FORMATS = {
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
@ -6,8 +6,8 @@ const FORMATS = {
function processImage(file: { path: string }) { function processImage(file: { path: string }) {
// this will overwrite the temp file // this will overwrite the temp file
return jimp.read(file.path).then(img => { return Jimp.read(file.path).then(img => {
return img.resize(300, jimp.AUTO).write(file.path) return img.resize({ w: 256 }).write(file.path as `${string}.${string}`)
}) })
} }

10
packages/worker/package.json
View File

@ -40,17 +40,17 @@
"dependencies": { "dependencies": {
"@budibase/backend-core": "0.0.0", "@budibase/backend-core": "0.0.0",
"@budibase/pro": "0.0.0", "@budibase/pro": "0.0.0",
"@budibase/shared-core": "0.0.0",
"@budibase/string-templates": "0.0.0", "@budibase/string-templates": "0.0.0",
"@budibase/types": "0.0.0", "@budibase/types": "0.0.0",
"@budibase/shared-core": "0.0.0", "@koa/router": "13.1.0",
"@koa/router": "8.0.8",
"@techpass/passport-openidconnect": "0.3.3", "@techpass/passport-openidconnect": "0.3.3",
"@types/global-agent": "2.1.1", "@types/global-agent": "2.1.1",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"dd-trace": "5.2.0", "dd-trace": "5.23.0",
"dotenv": "8.6.0", "dotenv": "8.6.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",
"ical-generator": "4.1.0", "ical-generator": "4.1.0",
@ -82,7 +82,7 @@
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/jsonwebtoken": "9.0.3", "@types/jsonwebtoken": "9.0.3",
"@types/koa": "2.13.4", "@types/koa": "2.13.4",
"@types/koa__router": "8.0.8", "@types/koa__router": "12.0.4",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/node": "^22.9.0", "@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",

1
packages/worker/src/api/controllers/global/users.ts
View File

@ -40,6 +40,7 @@ import {
import { checkAnyUserExists } from "../../../utilities/users" import { checkAnyUserExists } from "../../../utilities/users"
import { isEmailConfigured } from "../../../utilities/email" import { isEmailConfigured } from "../../../utilities/email"
import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core" import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core"
import crypto from "crypto"
const MAX_USERS_UPLOAD_LIMIT = 1000 const MAX_USERS_UPLOAD_LIMIT = 1000

3668
yarn.lock
View File

File diff suppressed because it is too large Load Diff