MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13359 from Budibase/reuse-containers 

Change the way we manage `testcontainers`
This commit is contained in:
Sam Rose 2024-04-03 14:38:26 +01:00 committed by GitHub
parent 03374f3abf e0ab2f7e0f
commit 86da22cbd7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 630 additions and 466 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.github/workflows/budibase_ci.yml vendored
View File

@ -91,6 +91,9 @@ jobs:
test-libraries:
runs-on: ubuntu-latest
env:
DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
REUSE_CONTAINERS: true
steps:
- name: Checkout repo
uses: actions/checkout@v4
@ -104,6 +107,14 @@ jobs:
with:
node-version: 20.x
cache: yarn
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb &
docker pull redis &
wait $(jobs -p)
- run: yarn --frozen-lockfile
- name: Test
run: |
@ -138,9 +149,10 @@ jobs:
fi
test-server:
runs-on: ubuntu-latest
runs-on: budi-tubby-tornado-quad-core-150gb
env:
DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
REUSE_CONTAINERS: true
steps:
- name: Checkout repo
uses: actions/checkout@v4
@ -157,13 +169,16 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull mcr.microsoft.com/mssql/server:2022-latest
docker pull mysql:8.3
docker pull postgres:16.1-bullseye
docker pull mongo:7.0-jammy
docker pull mariadb:lts
docker pull testcontainers/ryuk:0.5.1
docker pull budibase/couchdb
docker pull mcr.microsoft.com/mssql/server:2022-latest &
docker pull mysql:8.3 &
docker pull postgres:16.1-bullseye &
docker pull mongo:7.0-jammy &
docker pull mariadb:lts &
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb &
docker pull redis &
wait $(jobs -p)
- run: yarn --frozen-lockfile

58
globalSetup.ts
View File

@ -1,25 +1,47 @@
import { GenericContainer, Wait } from "testcontainers"
import path from "path"
import lockfile from "proper-lockfile"
export default async function setup() {
await new GenericContainer("budibase/couchdb")
.withExposedPorts(5984)
.withEnvironment({
COUCHDB_PASSWORD: "budibase",
COUCHDB_USER: "budibase",
})
.withCopyContentToContainer([
{
content: `
const lockPath = path.resolve(__dirname, "globalSetup.ts")
if (process.env.REUSE_CONTAINERS) {
// If you run multiple tests at the same time, it's possible for the CouchDB
// shared container to get started multiple times despite having an
// identical reuse hash. To avoid that, we do a filesystem-based lock so
// that only one globalSetup.ts is running at a time.
lockfile.lockSync(lockPath)
}
try {
let couchdb = new GenericContainer("budibase/couchdb")
.withExposedPorts(5984)
.withEnvironment({
COUCHDB_PASSWORD: "budibase",
COUCHDB_USER: "budibase",
})
.withCopyContentToContainer([
{
content: `
[log]
level = warn
`,
target: "/opt/couchdb/etc/local.d/test-couchdb.ini",
},
])
.withWaitStrategy(
Wait.forSuccessfulCommand(
"curl http://budibase:budibase@localhost:5984/_up"
).withStartupTimeout(20000)
)
.start()
target: "/opt/couchdb/etc/local.d/test-couchdb.ini",
},
])
.withWaitStrategy(
Wait.forSuccessfulCommand(
"curl http://budibase:budibase@localhost:5984/_up"
).withStartupTimeout(20000)
)
if (process.env.REUSE_CONTAINERS) {
couchdb = couchdb.withReuse()
}
await couchdb.start()
} finally {
if (process.env.REUSE_CONTAINERS) {
lockfile.unlockSync(lockPath)
}
}
}

2
package.json
View File

@ -7,6 +7,7 @@
"@babel/preset-env": "^7.22.5",
"@esbuild-plugins/tsconfig-paths": "^0.1.2",
"@types/node": "20.10.0",
"@types/proper-lockfile": "^4.1.4",
"@typescript-eslint/parser": "6.9.0",
"esbuild": "^0.18.17",
"esbuild-node-externals": "^1.8.0",
@ -23,6 +24,7 @@
"nx-cloud": "16.0.5",
"prettier": "2.8.8",
"prettier-plugin-svelte": "^2.3.0",
"proper-lockfile": "^4.1.2",
"svelte": "^4.2.10",
"svelte-eslint-parser": "^0.33.1",
"typescript": "5.2.2",

52
packages/backend-core/tests/core/utilities/testContainerUtils.ts
View File

@ -1,6 +1,7 @@
import { DatabaseImpl } from "../../../src/db"
import { execSync } from "child_process"
const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
interface ContainerInfo {
Command: string
CreatedAt: string
@ -19,7 +20,10 @@ interface ContainerInfo {
}
function getTestcontainers(): ContainerInfo[] {
return execSync("docker ps --format json")
// We use --format json to make sure the output is nice and machine-readable,
// and we use --no-trunc so that the command returns full container IDs so we
// can filter on them correctly.
return execSync("docker ps --format json --no-trunc")
.toString()
.split("\n")
.filter(x => x.length > 0)
@ -27,32 +31,55 @@ function getTestcontainers(): ContainerInfo[] {
.filter(x => x.Labels.includes("org.testcontainers=true"))
}
function getContainerByImage(image: string) {
return getTestcontainers().find(x => x.Image.startsWith(image))
export function getContainerByImage(image: string) {
const containers = getTestcontainers().filter(x => x.Image.startsWith(image))
if (containers.length > 1) {
let errorMessage = `Multiple containers found starting with image: "${image}"\n\n`
for (const container of containers) {
errorMessage += JSON.stringify(container, null, 2)
}
throw new Error(errorMessage)
}
return containers[0]
}
function getExposedPort(container: ContainerInfo, port: number) {
const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`))
if (!match) {
return undefined
export function getContainerById(id: string) {
return getTestcontainers().find(x => x.ID === id)
}
export interface Port {
host: number
container: number
}
export function getExposedV4Ports(container: ContainerInfo): Port[] {
let ports: Port[] = []
for (const match of container.Ports.matchAll(IPV4_PORT_REGEX)) {
ports.push({ host: parseInt(match[1]), container: parseInt(match[2]) })
}
return parseInt(match[1])
return ports
}
export function getExposedV4Port(container: ContainerInfo, port: number) {
return getExposedV4Ports(container).find(x => x.container === port)?.host
}
export function setupEnv(...envs: any[]) {
// We start couchdb in globalSetup.ts, in the root of the monorepo, so it
// should be relatively safe to look for it by its image name.
const couch = getContainerByImage("budibase/couchdb")
if (!couch) {
throw new Error("CouchDB container not found")
}
const couchPort = getExposedPort(couch, 5984)
const couchPort = getExposedV4Port(couch, 5984)
if (!couchPort) {
throw new Error("CouchDB port not found")
}
const configs = [
{ key: "COUCH_DB_PORT", value: `${couchPort}` },
{ key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` },
{ key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
]
for (const config of configs.filter(x => !!x.value)) {
@ -60,7 +87,4 @@ export function setupEnv(...envs: any[]) {
env._set(config.key, config.value)
}
}
// @ts-expect-error
DatabaseImpl.nano = undefined
}

4
packages/server/scripts/test.sh
View File

@ -4,8 +4,8 @@ set -e
if [[ -n $CI ]]
then
export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
echo "jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
else
# --maxWorkers performs better in development
export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"

2
packages/server/src/api/routes/public/tests/metrics.spec.js
View File

@ -1,7 +1,5 @@
const setup = require("../../tests/utilities")
jest.setTimeout(30000)
describe("/metrics", () => {
let request = setup.getRequest()
let config = setup.getConfig()

1
packages/server/src/api/routes/tests/appImport.spec.ts
View File

@ -1,7 +1,6 @@
import * as setup from "./utilities"
import path from "path"
jest.setTimeout(15000)
const PASSWORD = "testtest"
describe("/applications/:appId/import", () => {

2
packages/server/src/api/routes/tests/automation.spec.ts
View File

@ -23,8 +23,6 @@ let {
collectAutomation,
} = setup.structures
jest.setTimeout(30000)
describe("/automations", () => {
let request = setup.getRequest()
let config = setup.getConfig()

93
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -1,9 +1,10 @@
import { Datasource, Query, SourceName } from "@budibase/types"
import * as setup from "../utilities"
import { databaseTestProviders } from "../../../../integrations/tests/utils"
import pg from "pg"
import mysql from "mysql2/promise"
import mssql from "mssql"
import {
DatabaseName,
getDatasource,
rawQuery,
} from "../../../../integrations/tests/utils"
jest.unmock("pg")
@ -34,13 +35,16 @@ const createTableSQL: Record<string, string> = {
const insertSQL = `INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')`
const dropTableSQL = `DROP TABLE test_table;`
describe.each([
["postgres", databaseTestProviders.postgres],
["mysql", databaseTestProviders.mysql],
["mssql", databaseTestProviders.mssql],
["mariadb", databaseTestProviders.mariadb],
])("queries (%s)", (dbName, dsProvider) => {
describe.each(
[
DatabaseName.POSTGRES,
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
].map(name => [name, getDatasource(name)])
)("queries (%s)", (dbName, dsProvider) => {
const config = setup.getConfig()
let rawDatasource: Datasource
let datasource: Datasource
async function createQuery(query: Partial<Query>): Promise<Query> {
@ -57,62 +61,22 @@ describe.each([
return await config.api.query.save({ ...defaultQuery, ...query })
}
async function rawQuery(sql: string): Promise<any> {
// We re-fetch the datasource here because the one returned by
// config.api.datasource.create has the password field blanked out, and we
// need the password to connect to the database.
const ds = await dsProvider.datasource()
switch (ds.source) {
case SourceName.POSTGRES: {
const client = new pg.Client(ds.config!)
await client.connect()
try {
const { rows } = await client.query(sql)
return rows
} finally {
await client.end()
}
}
case SourceName.MYSQL: {
const con = await mysql.createConnection(ds.config!)
try {
const [rows] = await con.query(sql)
return rows
} finally {
con.end()
}
}
case SourceName.SQL_SERVER: {
const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
const client = await pool.connect()
try {
const { recordset } = await client.query(sql)
return recordset
} finally {
await pool.close()
}
}
}
}
beforeAll(async () => {
await config.init()
datasource = await config.api.datasource.create(
await dsProvider.datasource()
)
rawDatasource = await dsProvider
datasource = await config.api.datasource.create(rawDatasource)
})
beforeEach(async () => {
await rawQuery(createTableSQL[datasource.source])
await rawQuery(insertSQL)
await rawQuery(rawDatasource, createTableSQL[datasource.source])
await rawQuery(rawDatasource, insertSQL)
})
afterEach(async () => {
await rawQuery(dropTableSQL)
await rawQuery(rawDatasource, dropTableSQL)
})
afterAll(async () => {
await dsProvider.stop()
setup.afterAll()
})
@ -143,7 +107,10 @@ describe.each([
},
])
const rows = await rawQuery("SELECT * FROM test_table WHERE name = 'baz'")
const rows = await rawQuery(
rawDatasource,
"SELECT * FROM test_table WHERE name = 'baz'"
)
expect(rows).toHaveLength(1)
})
@ -171,6 +138,7 @@ describe.each([
expect(result.data).toEqual([{ created: true }])
const rows = await rawQuery(
rawDatasource,
`SELECT * FROM test_table WHERE birthday = '${date.toISOString()}'`
)
expect(rows).toHaveLength(1)
@ -202,6 +170,7 @@ describe.each([
expect(result.data).toEqual([{ created: true }])
const rows = await rawQuery(
rawDatasource,
`SELECT * FROM test_table WHERE name = '${notDateStr}'`
)
expect(rows).toHaveLength(1)
@ -338,7 +307,10 @@ describe.each([
},
])
const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
const rows = await rawQuery(
rawDatasource,
"SELECT * FROM test_table WHERE id = 1"
)
expect(rows).toEqual([
{ id: 1, name: "foo", birthday: null, number: null },
])
@ -406,7 +378,10 @@ describe.each([
},
])
const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
const rows = await rawQuery(
rawDatasource,
"SELECT * FROM test_table WHERE id = 1"
)
expect(rows).toHaveLength(0)
})
})
@ -443,7 +418,7 @@ describe.each([
} catch (err: any) {
error = err.message
}
if (dbName === "mssql") {
if (dbName === DatabaseName.SQL_SERVER) {
expect(error).toBeUndefined()
} else {
expect(error).toBeDefined()

29
packages/server/src/api/routes/tests/queries/mongodb.spec.ts
View File

@ -1,14 +1,17 @@
import { Datasource, Query } from "@budibase/types"
import * as setup from "../utilities"
import { databaseTestProviders } from "../../../../integrations/tests/utils"
import { MongoClient, type Collection, BSON } from "mongodb"
const collection = "test_collection"
import {
DatabaseName,
getDatasource,
} from "../../../../integrations/tests/utils"
import { MongoClient, type Collection, BSON, Db } from "mongodb"
import { generator } from "@budibase/backend-core/tests"
const expectValidId = expect.stringMatching(/^\w{24}$/)
const expectValidBsonObjectId = expect.any(BSON.ObjectId)
describe("/queries", () => {
let collection: string
let config = setup.getConfig()
let datasource: Datasource
@ -37,8 +40,7 @@ describe("/queries", () => {
async function withClient<T>(
callback: (client: MongoClient) => Promise<T>
): Promise<T> {
const ds = await databaseTestProviders.mongodb.datasource()
const client = new MongoClient(ds.config!.connectionString)
const client = new MongoClient(datasource.config!.connectionString)
await client.connect()
try {
return await callback(client)
@ -47,30 +49,33 @@ describe("/queries", () => {
}
}
async function withDb<T>(callback: (db: Db) => Promise<T>): Promise<T> {
return await withClient(async client => {
return await callback(client.db(datasource.config!.db))
})
}
async function withCollection<T>(
callback: (collection: Collection) => Promise<T>
): Promise<T> {
return await withClient(async client => {
const db = client.db(
(await databaseTestProviders.mongodb.datasource()).config!.db
)
return await withDb(async db => {
return await callback(db.collection(collection))
})
}
afterAll(async () => {
await databaseTestProviders.mongodb.stop()
setup.afterAll()
})
beforeAll(async () => {
await config.init()
datasource = await config.api.datasource.create(
await databaseTestProviders.mongodb.datasource()
await getDatasource(DatabaseName.MONGODB)
)
})
beforeEach(async () => {
collection = generator.guid()
await withCollection(async collection => {
await collection.insertMany([
{ name: "one" },

39
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -1,4 +1,4 @@
import { databaseTestProviders } from "../../../integrations/tests/utils"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import tk from "timekeeper"
import { outputProcessing } from "../../../utilities/rowProcessor"
@ -34,10 +34,10 @@ jest.unmock("pg")
describe.each([
["internal", undefined],
["postgres", databaseTestProviders.postgres],
["mysql", databaseTestProviders.mysql],
["mssql", databaseTestProviders.mssql],
["mariadb", databaseTestProviders.mariadb],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/rows (%s)", (__, dsProvider) => {
const isInternal = dsProvider === undefined
const config = setup.getConfig()
@ -49,23 +49,23 @@ describe.each([
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
datasource: await dsProvider.datasource(),
datasource: await dsProvider,
})
}
})
afterAll(async () => {
if (dsProvider) {
await dsProvider.stop()
}
setup.afterAll()
})
function saveTableRequest(
...overrides: Partial<SaveTableRequest>[]
// We omit the name field here because it's generated in the function with a
// high likelihood to be unique. Tests should not have any reason to control
// the table name they're writing to.
...overrides: Partial<Omit<SaveTableRequest, "name">>[]
): SaveTableRequest {
const req: SaveTableRequest = {
name: uuid.v4().substring(0, 16),
name: uuid.v4().substring(0, 10),
type: "table",
sourceType: datasource
? TableSourceType.EXTERNAL
@ -87,7 +87,10 @@ describe.each([
}
function defaultTable(
...overrides: Partial<SaveTableRequest>[]
// We omit the name field here because it's generated in the function with a
// high likelihood to be unique. Tests should not have any reason to control
// the table name they're writing to.
...overrides: Partial<Omit<SaveTableRequest, "name">>[]
): SaveTableRequest {
return saveTableRequest(
{
@ -194,7 +197,6 @@ describe.each([
const newTable = await config.api.table.save(
saveTableRequest({
name: "TestTableAuto",
schema: {
"Row ID": {
name: "Row ID",
@ -383,11 +385,9 @@ describe.each([
isInternal &&
it("doesn't allow creating in user table", async () => {
const userTableId = InternalTable.USER_METADATA
const response = await config.api.row.save(
userTableId,
InternalTable.USER_METADATA,
{
tableId: userTableId,
firstName: "Joe",
lastName: "Joe",
email: "joe@joe.com",
@ -462,7 +462,6 @@ describe.each([
table = await config.api.table.save(defaultTable())
otherTable = await config.api.table.save(
defaultTable({
name: "a",
schema: {
relationship: {
name: "relationship",
@ -898,8 +897,8 @@ describe.each([
let o2mTable: Table
let m2mTable: Table
beforeAll(async () => {
o2mTable = await config.api.table.save(defaultTable({ name: "o2m" }))
m2mTable = await config.api.table.save(defaultTable({ name: "m2m" }))
o2mTable = await config.api.table.save(defaultTable())
m2mTable = await config.api.table.save(defaultTable())
})
describe.each([
@ -1256,7 +1255,6 @@ describe.each([
otherTable = await config.api.table.save(defaultTable())
table = await config.api.table.save(
saveTableRequest({
name: "b",
schema: {
links: {
name: "links",
@ -1354,7 +1352,6 @@ describe.each([
const table = await config.api.table.save(
saveTableRequest({
name: "table",
schema: {
text: {
name: "text",

2
packages/server/src/api/routes/tests/user.spec.ts
View File

@ -3,8 +3,6 @@ import { checkPermissionsEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { UserMetadata } from "@budibase/types"
jest.setTimeout(30000)
jest.mock("../../../utilities/workerRequests", () => ({
getGlobalUsers: jest.fn(() => {
return {}

31
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -19,8 +19,7 @@ import {
ViewV2,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import * as uuid from "uuid"
import { databaseTestProviders } from "../../../integrations/tests/utils"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import { roles } from "@budibase/backend-core"
@ -30,10 +29,10 @@ jest.unmock("pg")
describe.each([
["internal", undefined],
["postgres", databaseTestProviders.postgres],
["mysql", databaseTestProviders.mysql],
["mssql", databaseTestProviders.mssql],
["mariadb", databaseTestProviders.mariadb],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/v2/views (%s)", (_, dsProvider) => {
const config = setup.getConfig()
const isInternal = !dsProvider
@ -42,10 +41,10 @@ describe.each([
let datasource: Datasource
function saveTableRequest(
...overrides: Partial<SaveTableRequest>[]
...overrides: Partial<Omit<SaveTableRequest, "name">>[]
): SaveTableRequest {
const req: SaveTableRequest = {
name: uuid.v4().substring(0, 16),
name: generator.guid().replaceAll("-", "").substring(0, 16),
type: "table",
sourceType: datasource
? TableSourceType.EXTERNAL
@ -90,16 +89,13 @@ describe.each([
if (dsProvider) {
datasource = await config.createDatasource({
datasource: await dsProvider.datasource(),
datasource: await dsProvider,
})
}
table = await config.api.table.save(priceTable())
})
afterAll(async () => {
if (dsProvider) {
await dsProvider.stop()
}
setup.afterAll()
})
@ -231,7 +227,7 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: "View A",
name: generator.guid(),
})
})
@ -307,12 +303,13 @@ describe.each([
it("can update an existing view name", async () => {
const tableId = table._id!
await config.api.viewV2.update({ ...view, name: "View B" })
const newName = generator.guid()
await config.api.viewV2.update({ ...view, name: newName })
expect(await config.api.table.get(tableId)).toEqual(
expect.objectContaining({
views: {
"View B": { ...view, name: "View B", schema: expect.anything() },
[newName]: { ...view, name: newName, schema: expect.anything() },
},
})
)
@ -507,7 +504,6 @@ describe.each([
it("views have extra data trimmed", async () => {
const table = await config.api.table.save(
saveTableRequest({
name: "orders",
schema: {
Country: {
type: FieldType.STRING,
@ -523,7 +519,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: uuid.v4(),
name: generator.guid(),
schema: {
Country: {
visible: true,
@ -853,7 +849,6 @@ describe.each([
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
name: `users_${uuid.v4()}`,
type: "table",
schema: {
name: {

133
packages/server/src/integration-test/mysql.spec.ts
View File

@ -3,7 +3,6 @@ import {
generateMakeRequest,
MakeRequestResponse,
} from "../api/routes/public/tests/utils"
import { v4 as uuidv4 } from "uuid"
import * as setup from "../api/routes/tests/utilities"
import {
Datasource,
@ -12,12 +11,23 @@ import {
TableRequest,
TableSourceType,
} from "@budibase/types"
import { databaseTestProviders } from "../integrations/tests/utils"
import mysql from "mysql2/promise"
import {
DatabaseName,
getDatasource,
rawQuery,
} from "../integrations/tests/utils"
import { builderSocket } from "../websockets"
import { generator } from "@budibase/backend-core/tests"
// @ts-ignore
fetch.mockSearch()
function uniqueTableName(length?: number): string {
return generator
.guid()
.replaceAll("-", "_")
.substring(0, length || 10)
}
const config = setup.getConfig()!
jest.mock("../websockets", () => ({
@ -37,7 +47,8 @@ jest.mock("../websockets", () => ({
describe("mysql integrations", () => {
let makeRequest: MakeRequestResponse,
mysqlDatasource: Datasource,
rawDatasource: Datasource,
datasource: Datasource,
primaryMySqlTable: Table
beforeAll(async () => {
@ -46,18 +57,13 @@ describe("mysql integrations", () => {
makeRequest = generateMakeRequest(apiKey, true)
mysqlDatasource = await config.api.datasource.create(
await databaseTestProviders.mysql.datasource()
)
})
afterAll(async () => {
await databaseTestProviders.mysql.stop()
rawDatasource = await getDatasource(DatabaseName.MYSQL)
datasource = await config.api.datasource.create(rawDatasource)
})
beforeEach(async () => {
primaryMySqlTable = await config.createTable({
name: uuidv4(),
name: uniqueTableName(),
type: "table",
primary: ["id"],
schema: {
@ -79,7 +85,7 @@ describe("mysql integrations", () => {
type: FieldType.NUMBER,
},
},
sourceId: mysqlDatasource._id,
sourceId: datasource._id,
sourceType: TableSourceType.EXTERNAL,
})
})
@ -87,18 +93,15 @@ describe("mysql integrations", () => {
afterAll(config.end)
it("validate table schema", async () => {
const res = await makeRequest(
"get",
`/api/datasources/${mysqlDatasource._id}`
)
const res = await makeRequest("get", `/api/datasources/${datasource._id}`)
expect(res.status).toBe(200)
expect(res.body).toEqual({
config: {
database: "mysql",
host: mysqlDatasource.config!.host,
database: expect.any(String),
host: datasource.config!.host,
password: "--secret-value--",
port: mysqlDatasource.config!.port,
port: datasource.config!.port,
user: "root",
},
plus: true,
@ -117,7 +120,7 @@ describe("mysql integrations", () => {
it("should be able to verify the connection", async () => {
await config.api.datasource.verify(
{
datasource: await databaseTestProviders.mysql.datasource(),
datasource: rawDatasource,
},
{
body: {
@ -128,13 +131,12 @@ describe("mysql integrations", () => {
})
it("should state an invalid datasource cannot connect", async () => {
const dbConfig = await databaseTestProviders.mysql.datasource()
await config.api.datasource.verify(
{
datasource: {
...dbConfig,
...rawDatasource,
config: {
...dbConfig.config,
...rawDatasource.config,
password: "wrongpassword",
},
},
@ -154,7 +156,7 @@ describe("mysql integrations", () => {
it("should fetch information about mysql datasource", async () => {
const primaryName = primaryMySqlTable.name
const response = await makeRequest("post", "/api/datasources/info", {
datasource: mysqlDatasource,
datasource: datasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
@ -163,40 +165,38 @@ describe("mysql integrations", () => {
})
describe("Integration compatibility with mysql search_path", () => {
let client: mysql.Connection, pathDatasource: Datasource
const database = "test1"
const database2 = "test-2"
let datasource: Datasource, rawDatasource: Datasource
const database = generator.guid()
const database2 = generator.guid()
beforeAll(async () => {
const dsConfig = await databaseTestProviders.mysql.datasource()
const dbConfig = dsConfig.config!
rawDatasource = await getDatasource(DatabaseName.MYSQL)
client = await mysql.createConnection(dbConfig)
await client.query(`CREATE DATABASE \`${database}\`;`)
await client.query(`CREATE DATABASE \`${database2}\`;`)
await rawQuery(rawDatasource, `CREATE DATABASE \`${database}\`;`)
await rawQuery(rawDatasource, `CREATE DATABASE \`${database2}\`;`)
const pathConfig: any = {
...dsConfig,
...rawDatasource,
config: {
...dbConfig,
...rawDatasource.config!,
database,
},
}
pathDatasource = await config.api.datasource.create(pathConfig)
datasource = await config.api.datasource.create(pathConfig)
})
afterAll(async () => {
await client.query(`DROP DATABASE \`${database}\`;`)
await client.query(`DROP DATABASE \`${database2}\`;`)
await client.end()
await rawQuery(rawDatasource, `DROP DATABASE \`${database}\`;`)
await rawQuery(rawDatasource, `DROP DATABASE \`${database2}\`;`)
})
it("discovers tables from any schema in search path", async () => {
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
)
const response = await makeRequest("post", "/api/datasources/info", {
datasource: pathDatasource,
datasource: datasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
@ -207,15 +207,17 @@ describe("mysql integrations", () => {
it("does not mix columns from different tables", async () => {
const repeated_table_name = "table_same_name"
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
)
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
)
const response = await makeRequest(
"post",
`/api/datasources/${pathDatasource._id}/schema`,
`/api/datasources/${datasource._id}/schema`,
{
tablesFilter: [repeated_table_name],
}
@ -231,30 +233,14 @@ describe("mysql integrations", () => {
})
describe("POST /api/tables/", () => {
let client: mysql.Connection
const emitDatasourceUpdateMock = jest.fn()
beforeEach(async () => {
client = await mysql.createConnection(
(
await databaseTestProviders.mysql.datasource()
).config!
)
mysqlDatasource = await config.api.datasource.create(
await databaseTestProviders.mysql.datasource()
)
})
afterEach(async () => {
await client.end()
})
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: "table",
sourceId: mysqlDatasource._id!,
name: uniqueTableName(),
sourceId: datasource._id!,
primary: ["id"],
schema: {
id: {
@ -301,14 +287,16 @@ describe("mysql integrations", () => {
},
},
created: true,
_id: `${mysqlDatasource._id}__table`,
_id: `${datasource._id}__${addColumnToTable.name}`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities!["table"]).toEqual(expectedTable)
expect(emittedDatasource.entities![expectedTable.name]).toEqual(
expectedTable
)
})
it("will rename a column", async () => {
@ -346,17 +334,18 @@ describe("mysql integrations", () => {
"/api/tables/",
renameColumnOnTable
)
mysqlDatasource = (
await makeRequest(
"post",
`/api/datasources/${mysqlDatasource._id}/schema`
)
const ds = (
await makeRequest("post", `/api/datasources/${datasource._id}/schema`)
).body.datasource
expect(response.status).toEqual(200)
expect(
Object.keys(mysqlDatasource.entities![primaryMySqlTable.name].schema)
).toEqual(["id", "name", "description", "age"])
expect(Object.keys(ds.entities![primaryMySqlTable.name].schema)).toEqual([
"id",
"name",
"description",
"age",
])
})
})
})

126
packages/server/src/integration-test/postgres.spec.ts
View File

@ -16,8 +16,12 @@ import {
import _ from "lodash"
import { generator } from "@budibase/backend-core/tests"
import { utils } from "@budibase/backend-core"
import { databaseTestProviders } from "../integrations/tests/utils"
import { Client } from "pg"
import {
DatabaseName,
getDatasource,
rawQuery,
} from "../integrations/tests/utils"
// @ts-ignore
fetch.mockSearch()
@ -28,7 +32,8 @@ jest.mock("../websockets")
describe("postgres integrations", () => {
let makeRequest: MakeRequestResponse,
postgresDatasource: Datasource,
rawDatasource: Datasource,
datasource: Datasource,
primaryPostgresTable: Table,
oneToManyRelationshipInfo: ForeignTableInfo,
manyToOneRelationshipInfo: ForeignTableInfo,
@ -40,19 +45,17 @@ describe("postgres integrations", () => {
makeRequest = generateMakeRequest(apiKey, true)
postgresDatasource = await config.api.datasource.create(
await databaseTestProviders.postgres.datasource()
)
})
afterAll(async () => {
await databaseTestProviders.postgres.stop()
rawDatasource = await getDatasource(DatabaseName.POSTGRES)
datasource = await config.api.datasource.create(rawDatasource)
})
beforeEach(async () => {
async function createAuxTable(prefix: string) {
return await config.createTable({
name: `${prefix}_${generator.word({ length: 6 })}`,
name: `${prefix}_${generator
.guid()
.replaceAll("-", "")
.substring(0, 6)}`,
type: "table",
primary: ["id"],
primaryDisplay: "title",
@ -67,7 +70,7 @@ describe("postgres integrations", () => {
type: FieldType.STRING,
},
},
sourceId: postgresDatasource._id,
sourceId: datasource._id,
sourceType: TableSourceType.EXTERNAL,
})
}
@ -89,7 +92,7 @@ describe("postgres integrations", () => {
}
primaryPostgresTable = await config.createTable({
name: `p_${generator.word({ length: 6 })}`,
name: `p_${generator.guid().replaceAll("-", "").substring(0, 6)}`,
type: "table",
primary: ["id"],
schema: {
@ -144,7 +147,7 @@ describe("postgres integrations", () => {
main: true,
},
},
sourceId: postgresDatasource._id,
sourceId: datasource._id,
sourceType: TableSourceType.EXTERNAL,
})
})
@ -251,7 +254,7 @@ describe("postgres integrations", () => {
async function createDefaultPgTable() {
return await config.createTable({
name: generator.word({ length: 10 }),
name: generator.guid().replaceAll("-", "").substring(0, 10),
type: "table",
primary: ["id"],
schema: {
@ -261,7 +264,7 @@ describe("postgres integrations", () => {
autocolumn: true,
},
},
sourceId: postgresDatasource._id,
sourceId: datasource._id,
sourceType: TableSourceType.EXTERNAL,
})
}
@ -299,19 +302,16 @@ describe("postgres integrations", () => {
}
it("validate table schema", async () => {
const res = await makeRequest(
"get",
`/api/datasources/${postgresDatasource._id}`
)
const res = await makeRequest("get", `/api/datasources/${datasource._id}`)
expect(res.status).toBe(200)
expect(res.body).toEqual({
config: {
ca: false,
database: "postgres",
host: postgresDatasource.config!.host,
database: expect.any(String),
host: datasource.config!.host,
password: "--secret-value--",
port: postgresDatasource.config!.port,
port: datasource.config!.port,
rejectUnauthorized: false,
schema: "public",
ssl: false,
@ -1043,7 +1043,7 @@ describe("postgres integrations", () => {
it("should be able to verify the connection", async () => {
await config.api.datasource.verify(
{
datasource: await databaseTestProviders.postgres.datasource(),
datasource: await getDatasource(DatabaseName.POSTGRES),
},
{
body: {
@ -1054,7 +1054,7 @@ describe("postgres integrations", () => {
})
it("should state an invalid datasource cannot connect", async () => {
const dbConfig = await databaseTestProviders.postgres.datasource()
const dbConfig = await getDatasource(DatabaseName.POSTGRES)
await config.api.datasource.verify(
{
datasource: {
@ -1079,7 +1079,7 @@ describe("postgres integrations", () => {
it("should fetch information about postgres datasource", async () => {
const primaryName = primaryPostgresTable.name
const response = await makeRequest("post", "/api/datasources/info", {
datasource: postgresDatasource,
datasource: datasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
@ -1088,86 +1088,88 @@ describe("postgres integrations", () => {
})
describe("POST /api/datasources/:datasourceId/schema", () => {
let client: Client
let tableName: string
beforeEach(async () => {
client = new Client(
(await databaseTestProviders.postgres.datasource()).config!
)
await client.connect()
tableName = generator.guid().replaceAll("-", "").substring(0, 10)
})
afterEach(async () => {
await client.query(`DROP TABLE IF EXISTS "table"`)
await client.end()
await rawQuery(rawDatasource, `DROP TABLE IF EXISTS "${tableName}"`)
})
it("recognises when a table has no primary key", async () => {
await client.query(`CREATE TABLE "table" (id SERIAL)`)
await rawQuery(rawDatasource, `CREATE TABLE "${tableName}" (id SERIAL)`)
const response = await makeRequest(
"post",
`/api/datasources/${postgresDatasource._id}/schema`
`/api/datasources/${datasource._id}/schema`
)
expect(response.body.errors).toEqual({
table: "Table must have a primary key.",
[tableName]: "Table must have a primary key.",
})
})
it("recognises when a table is using a reserved column name", async () => {
await client.query(`CREATE TABLE "table" (_id SERIAL PRIMARY KEY) `)
await rawQuery(
rawDatasource,
`CREATE TABLE "${tableName}" (_id SERIAL PRIMARY KEY) `
)
const response = await makeRequest(
"post",
`/api/datasources/${postgresDatasource._id}/schema`
`/api/datasources/${datasource._id}/schema`
)
expect(response.body.errors).toEqual({
table: "Table contains invalid columns.",
[tableName]: "Table contains invalid columns.",
})
})
})
describe("Integration compatibility with postgres search_path", () => {
let client: Client, pathDatasource: Datasource
const schema1 = "test1",
schema2 = "test-2"
let rawDatasource: Datasource,
datasource: Datasource,
schema1: string,
schema2: string
beforeAll(async () => {
const dsConfig = await databaseTestProviders.postgres.datasource()
const dbConfig = dsConfig.config!
beforeEach(async () => {
schema1 = generator.guid().replaceAll("-", "")
schema2 = generator.guid().replaceAll("-", "")
client = new Client(dbConfig)
await client.connect()
await client.query(`CREATE SCHEMA "${schema1}";`)
await client.query(`CREATE SCHEMA "${schema2}";`)
rawDatasource = await getDatasource(DatabaseName.POSTGRES)
const dbConfig = rawDatasource.config!
await rawQuery(rawDatasource, `CREATE SCHEMA "${schema1}";`)
await rawQuery(rawDatasource, `CREATE SCHEMA "${schema2}";`)
const pathConfig: any = {
...dsConfig,
...rawDatasource,
config: {
...dbConfig,
schema: `${schema1}, ${schema2}`,
},
}
pathDatasource = await config.api.datasource.create(pathConfig)
datasource = await config.api.datasource.create(pathConfig)
})
afterAll(async () => {
await client.query(`DROP SCHEMA "${schema1}" CASCADE;`)
await client.query(`DROP SCHEMA "${schema2}" CASCADE;`)
await client.end()
afterEach(async () => {
await rawQuery(rawDatasource, `DROP SCHEMA "${schema1}" CASCADE;`)
await rawQuery(rawDatasource, `DROP SCHEMA "${schema2}" CASCADE;`)
})
it("discovers tables from any schema in search path", async () => {
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
)
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
)
const response = await makeRequest("post", "/api/datasources/info", {
datasource: pathDatasource,
datasource: datasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
@ -1178,15 +1180,17 @@ describe("postgres integrations", () => {
it("does not mix columns from different tables", async () => {
const repeated_table_name = "table_same_name"
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
)
await client.query(
await rawQuery(
rawDatasource,
`CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
)
const response = await makeRequest(
"post",
`/api/datasources/${pathDatasource._id}/schema`,
`/api/datasources/${datasource._id}/schema`,
{
tablesFilter: [repeated_table_name],
}

91
packages/server/src/integrations/tests/utils/index.ts
View File

@ -1,25 +1,90 @@
jest.unmock("pg")
import { Datasource } from "@budibase/types"
import { Datasource, SourceName } from "@budibase/types"
import * as postgres from "./postgres"
import * as mongodb from "./mongodb"
import * as mysql from "./mysql"
import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import { StartedTestContainer } from "testcontainers"
import { GenericContainer } from "testcontainers"
import { testContainerUtils } from "@budibase/backend-core/tests"
jest.setTimeout(30000)
export type DatasourceProvider = () => Promise<Datasource>
export interface DatabaseProvider {
start(): Promise<StartedTestContainer>
stop(): Promise<void>
datasource(): Promise<Datasource>
export enum DatabaseName {
POSTGRES = "postgres",
MONGODB = "mongodb",
MYSQL = "mysql",
SQL_SERVER = "mssql",
MARIADB = "mariadb",
}
export const databaseTestProviders = {
postgres,
mongodb,
mysql,
mssql,
mariadb,
const providers: Record<DatabaseName, DatasourceProvider> = {
[DatabaseName.POSTGRES]: postgres.getDatasource,
[DatabaseName.MONGODB]: mongodb.getDatasource,
[DatabaseName.MYSQL]: mysql.getDatasource,
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
[DatabaseName.MARIADB]: mariadb.getDatasource,
}
export function getDatasourceProviders(
...sourceNames: DatabaseName[]
): Promise<Datasource>[] {
return sourceNames.map(sourceName => providers[sourceName]())
}
export function getDatasourceProvider(
sourceName: DatabaseName
): DatasourceProvider {
return providers[sourceName]
}
export function getDatasource(sourceName: DatabaseName): Promise<Datasource> {
return providers[sourceName]()
}
export async function getDatasources(
...sourceNames: DatabaseName[]
): Promise<Datasource[]> {
return Promise.all(sourceNames.map(sourceName => providers[sourceName]()))
}
export async function rawQuery(ds: Datasource, sql: string): Promise<any> {
switch (ds.source) {
case SourceName.POSTGRES: {
return postgres.rawQuery(ds, sql)
}
case SourceName.MYSQL: {
return mysql.rawQuery(ds, sql)
}
case SourceName.SQL_SERVER: {
return mssql.rawQuery(ds, sql)
}
default: {
throw new Error(`Unsupported source: ${ds.source}`)
}
}
}
export async function startContainer(container: GenericContainer) {
if (process.env.REUSE_CONTAINERS) {
container = container.withReuse()
}
const startedContainer = await container.start()
const info = testContainerUtils.getContainerById(startedContainer.getId())
if (!info) {
throw new Error("Container not found")
}
// Some Docker runtimes, when you expose a port, will bind it to both
// 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
// addresses are not shared, and testcontainers will sometimes give you back
// the ipv6 port. There's no way to know that this has happened, and if you
// try to then connect to `localhost:port` you may attempt to bind to the v4
// address which could be unbound or even an entirely different container. For
// that reason, we don't use testcontainers' `getExposedPort` function,
// preferring instead our own method that guaranteed v4 ports.
return testContainerUtils.getExposedV4Ports(info)
}

61
packages/server/src/integrations/tests/utils/mariadb.ts
View File

@ -1,8 +1,11 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { GenericContainer, Wait } from "testcontainers"
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
import { rawQuery } from "./mysql"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
let container: StartedTestContainer | undefined
let ports: Promise<testContainerUtils.Port[]>
class MariaDBWaitStrategy extends AbstractWaitStrategy {
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@ -21,38 +24,38 @@ class MariaDBWaitStrategy extends AbstractWaitStrategy {
}
}
export async function start(): Promise<StartedTestContainer> {
return await new GenericContainer("mariadb:lts")
.withExposedPorts(3306)
.withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MariaDBWaitStrategy())
.start()
}
export async function datasource(): Promise<Datasource> {
if (!container) {
container = await start()
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer("mariadb:lts")
.withExposedPorts(3306)
.withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MariaDBWaitStrategy())
)
}
const host = container.getHost()
const port = container.getMappedPort(3306)
return {
const port = (await ports).find(x => x.container === 3306)?.host
if (!port) {
throw new Error("MariaDB port not found")
}
const config = {
host: "127.0.0.1",
port,
user: "root",
password: "password",
database: "mysql",
}
const datasource = {
type: "datasource_plus",
source: SourceName.MYSQL,
plus: true,
config: {
host,
port,
user: "root",
password: "password",
database: "mysql",
},
config,
}
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
}
const database = generator.guid().replaceAll("-", "")
await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
datasource.config.database = database
return datasource
}

56
packages/server/src/integrations/tests/utils/mongodb.ts
View File

@ -1,43 +1,39 @@
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { GenericContainer, Wait } from "testcontainers"
import { startContainer } from "."
let container: StartedTestContainer | undefined
let ports: Promise<testContainerUtils.Port[]>
export async function start(): Promise<StartedTestContainer> {
return await new GenericContainer("mongo:7.0-jammy")
.withExposedPorts(27017)
.withEnvironment({
MONGO_INITDB_ROOT_USERNAME: "mongo",
MONGO_INITDB_ROOT_PASSWORD: "password",
})
.withWaitStrategy(
Wait.forSuccessfulCommand(
`mongosh --eval "db.version()"`
).withStartupTimeout(10000)
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer("mongo:7.0-jammy")
.withExposedPorts(27017)
.withEnvironment({
MONGO_INITDB_ROOT_USERNAME: "mongo",
MONGO_INITDB_ROOT_PASSWORD: "password",
})
.withWaitStrategy(
Wait.forSuccessfulCommand(
`mongosh --eval "db.version()"`
).withStartupTimeout(10000)
)
)
.start()
}
export async function datasource(): Promise<Datasource> {
if (!container) {
container = await start()
}
const host = container.getHost()
const port = container.getMappedPort(27017)
const port = (await ports).find(x => x.container === 27017)
if (!port) {
throw new Error("MongoDB port not found")
}
return {
type: "datasource",
source: SourceName.MONGODB,
plus: false,
config: {
connectionString: `mongodb://mongo:password@${host}:${port}`,
db: "mongo",
connectionString: `mongodb://mongo:password@127.0.0.1:${port.host}`,
db: generator.guid(),
},
}
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
}
}

83
packages/server/src/integrations/tests/utils/mssql.ts
View File

@ -1,43 +1,41 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { GenericContainer, Wait } from "testcontainers"
import mssql from "mssql"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
let container: StartedTestContainer | undefined
let ports: Promise<testContainerUtils.Port[]>
export async function start(): Promise<StartedTestContainer> {
return await new GenericContainer(
"mcr.microsoft.com/mssql/server:2022-latest"
)
.withExposedPorts(1433)
.withEnvironment({
ACCEPT_EULA: "Y",
MSSQL_SA_PASSWORD: "Password_123",
// This is important, as Microsoft allow us to use the "Developer" edition
// of SQL Server for development and testing purposes. We can't use other
// versions without a valid license, and we cannot use the Developer
// version in production.
MSSQL_PID: "Developer",
})
.withWaitStrategy(
Wait.forSuccessfulCommand(
"/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
)
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer("mcr.microsoft.com/mssql/server:2022-latest")
.withExposedPorts(1433)
.withEnvironment({
ACCEPT_EULA: "Y",
MSSQL_SA_PASSWORD: "Password_123",
// This is important, as Microsoft allow us to use the "Developer" edition
// of SQL Server for development and testing purposes. We can't use other
// versions without a valid license, and we cannot use the Developer
// version in production.
MSSQL_PID: "Developer",
})
.withWaitStrategy(
Wait.forSuccessfulCommand(
"/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
)
)
)
.start()
}
export async function datasource(): Promise<Datasource> {
if (!container) {
container = await start()
}
const host = container.getHost()
const port = container.getMappedPort(1433)
return {
const port = (await ports).find(x => x.container === 1433)?.host
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.SQL_SERVER,
plus: true,
config: {
server: host,
server: "127.0.0.1",
port,
user: "sa",
password: "Password_123",
@ -46,11 +44,28 @@ export async function datasource(): Promise<Datasource> {
},
},
}
const database = generator.guid().replaceAll("-", "")
await rawQuery(datasource, `CREATE DATABASE "${database}"`)
datasource.config!.database = database
return datasource
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
export async function rawQuery(ds: Datasource, sql: string) {
if (!ds.config) {
throw new Error("Datasource config is missing")
}
if (ds.source !== SourceName.SQL_SERVER) {
throw new Error("Datasource source is not SQL Server")
}
const pool = new mssql.ConnectionPool(ds.config! as mssql.config)
const client = await pool.connect()
try {
const { recordset } = await client.query(sql)
return recordset
} finally {
await pool.close()
}
}

57
packages/server/src/integrations/tests/utils/mysql.ts
View File

@ -1,8 +1,11 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { GenericContainer, Wait } from "testcontainers"
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
import mysql from "mysql2/promise"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
let container: StartedTestContainer | undefined
let ports: Promise<testContainerUtils.Port[]>
class MySQLWaitStrategy extends AbstractWaitStrategy {
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
@ -24,38 +27,50 @@ class MySQLWaitStrategy extends AbstractWaitStrategy {
}
}
export async function start(): Promise<StartedTestContainer> {
return await new GenericContainer("mysql:8.3")
.withExposedPorts(3306)
.withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
.start()
}
export async function datasource(): Promise<Datasource> {
if (!container) {
container = await start()
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer("mysql:8.3")
.withExposedPorts(3306)
.withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
)
}
const host = container.getHost()
const port = container.getMappedPort(3306)
return {
const port = (await ports).find(x => x.container === 3306)?.host
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.MYSQL,
plus: true,
config: {
host,
host: "127.0.0.1",
port,
user: "root",
password: "password",
database: "mysql",
},
}
const database = generator.guid().replaceAll("-", "")
await rawQuery(datasource, `CREATE DATABASE \`${database}\``)
datasource.config!.database = database
return datasource
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
export async function rawQuery(ds: Datasource, sql: string) {
if (!ds.config) {
throw new Error("Datasource config is missing")
}
if (ds.source !== SourceName.MYSQL) {
throw new Error("Datasource source is not MySQL")
}
const connection = await mysql.createConnection(ds.config)
try {
const [rows] = await connection.query(sql)
return rows
} finally {
connection.end()
}
}

65
packages/server/src/integrations/tests/utils/postgres.ts
View File

@ -1,33 +1,33 @@
import { Datasource, SourceName } from "@budibase/types"
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { GenericContainer, Wait } from "testcontainers"
import pg from "pg"
import { generator, testContainerUtils } from "@budibase/backend-core/tests"
import { startContainer } from "."
let container: StartedTestContainer | undefined
let ports: Promise<testContainerUtils.Port[]>
export async function start(): Promise<StartedTestContainer> {
return await new GenericContainer("postgres:16.1-bullseye")
.withExposedPorts(5432)
.withEnvironment({ POSTGRES_PASSWORD: "password" })
.withWaitStrategy(
Wait.forSuccessfulCommand(
"pg_isready -h localhost -p 5432"
).withStartupTimeout(10000)
export async function getDatasource(): Promise<Datasource> {
if (!ports) {
ports = startContainer(
new GenericContainer("postgres:16.1-bullseye")
.withExposedPorts(5432)
.withEnvironment({ POSTGRES_PASSWORD: "password" })
.withWaitStrategy(
Wait.forSuccessfulCommand(
"pg_isready -h localhost -p 5432"
).withStartupTimeout(10000)
)
)
.start()
}
export async function datasource(): Promise<Datasource> {
if (!container) {
container = await start()
}
const host = container.getHost()
const port = container.getMappedPort(5432)
return {
const port = (await ports).find(x => x.container === 5432)?.host
const datasource: Datasource = {
type: "datasource_plus",
source: SourceName.POSTGRES,
plus: true,
config: {
host,
host: "127.0.0.1",
port,
database: "postgres",
user: "postgres",
@ -38,11 +38,28 @@ export async function datasource(): Promise<Datasource> {
ca: false,
},
}
const database = generator.guid().replaceAll("-", "")
await rawQuery(datasource, `CREATE DATABASE "${database}"`)
datasource.config!.database = database
return datasource
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
export async function rawQuery(ds: Datasource, sql: string) {
if (!ds.config) {
throw new Error("Datasource config is missing")
}
if (ds.source !== SourceName.POSTGRES) {
throw new Error("Datasource source is not Postgres")
}
const client = new pg.Client(ds.config)
await client.connect()
try {
const { rows } = await client.query(sql)
return rows
} finally {
await client.end()
}
}

2
packages/server/src/migrations/tests/index.spec.ts
View File

@ -25,8 +25,6 @@ const clearMigrations = async () => {
}
}
jest.setTimeout(10000)
describe("migrations", () => {
const config = new TestConfig()

2
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -17,8 +17,6 @@ import {
generator,
} from "@budibase/backend-core/tests"
jest.setTimeout(30000)
describe("external search", () => {
const config = new TestConfiguration()

10
packages/server/src/tests/jestSetup.ts
View File

@ -2,17 +2,11 @@ import env from "../environment"
import { env as coreEnv, timers } from "@budibase/backend-core"
import { testContainerUtils } from "@budibase/backend-core/tests"
if (!process.env.DEBUG) {
global.console.log = jest.fn() // console.log are ignored in tests
global.console.warn = jest.fn() // console.warn are ignored in tests
}
if (!process.env.CI) {
// set a longer timeout in dev for debugging
// 100 seconds
// set a longer timeout in dev for debugging 100 seconds
jest.setTimeout(100 * 1000)
} else {
jest.setTimeout(10 * 1000)
jest.setTimeout(30 * 1000)
}
testContainerUtils.setupEnv(env, coreEnv)

52
packages/server/src/tests/utilities/api/base.ts
View File

@ -1,6 +1,7 @@
import TestConfiguration from "../TestConfiguration"
import { SuperTest, Test, Response } from "supertest"
import request, { SuperTest, Test, Response } from "supertest"
import { ReadStream } from "fs"
import { getServer } from "../../../app"
type Headers = Record<string, string | string[] | undefined>
type Method = "get" | "post" | "put" | "patch" | "delete"
@ -76,7 +77,8 @@ export abstract class TestAPI {
protected _requestRaw = async (
method: "get" | "post" | "put" | "patch" | "delete",
url: string,
opts?: RequestOpts
opts?: RequestOpts,
attempt = 0
): Promise<Response> => {
const {
headers = {},
@ -107,26 +109,29 @@ export abstract class TestAPI {
const headersFn = publicUser
? this.config.publicHeaders.bind(this.config)
: this.config.defaultHeaders.bind(this.config)
let request = this.request[method](url).set(
const app = getServer()
let req = request(app)[method](url)
req = req.set(
headersFn({
"x-budibase-include-stacktrace": "true",
})
)
if (headers) {
request = request.set(headers)
req = req.set(headers)
}
if (body) {
request = request.send(body)
req = req.send(body)
}
for (const [key, value] of Object.entries(fields)) {
request = request.field(key, value)
req = req.field(key, value)
}
for (const [key, value] of Object.entries(files)) {
if (isAttachedFile(value)) {
request = request.attach(key, value.file, value.name)
req = req.attach(key, value.file, value.name)
} else {
request = request.attach(key, value as any)
req = req.attach(key, value as any)
}
}
if (expectations?.headers) {
@ -136,11 +141,25 @@ export abstract class TestAPI {
`Got an undefined expected value for header "${key}", if you want to check for the absence of a header, use headersNotPresent`
)
}
request = request.expect(key, value as any)
req = req.expect(key, value as any)
}
}
return await request
try {
return await req
} catch (e: any) {
// We've found that occasionally the connection between supertest and the
// server supertest starts gets reset. Not sure why, but retrying it
// appears to work. I don't particularly like this, but it's better than
// flakiness.
if (e.code === "ECONNRESET") {
if (attempt > 2) {
throw e
}
return await this._requestRaw(method, url, opts, attempt + 1)
}
throw e
}
}
protected _checkResponse = (
@ -170,7 +189,18 @@ export abstract class TestAPI {
}
}
throw new Error(message)
if (response.error) {
// Sometimes the error can be between supertest and the app, and when
// that happens response.error is sometimes populated with `text` that
// gives more detail about the error. The `message` is almost always
// useless from what I've seen.
if (response.error.text) {
response.error.message = response.error.text
}
throw new Error(message, { cause: response.error })
} else {
throw new Error(message)
}
}
if (expectations?.headersNotPresent) {

12
yarn.lock
View File

@ -5875,6 +5875,13 @@
"@types/pouchdb-node" "*"
"@types/pouchdb-replication" "*"
"@types/proper-lockfile@^4.1.4":
version "4.1.4"
resolved "https://registry.yarnpkg.com/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz#cd9fab92bdb04730c1ada542c356f03620f84008"
integrity sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==
dependencies:
"@types/retry" "*"
"@types/qs@*":
version "6.9.7"
resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
@ -5937,6 +5944,11 @@
dependencies:
"@types/node" "*"
"@types/retry@*":
version "0.12.5"
resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.5.tgz#f090ff4bd8d2e5b940ff270ab39fd5ca1834a07e"
integrity sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==
"@types/rimraf@^3.0.2":
version "3.0.2"
resolved "https://registry.yarnpkg.com/@types/rimraf/-/rimraf-3.0.2.tgz#a63d175b331748e5220ad48c901d7bbf1f44eef8"