Merge branch 'master' into create-secret-key-once

This commit is contained in:
Sam Rose 2024-11-18 11:28:58 +00:00 committed by GitHub
commit 1bec450623
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
46 changed files with 14483 additions and 14195 deletions

View File

@ -147,7 +147,10 @@ jobs:
fi
test-server:
runs-on: budi-tubby-tornado-quad-core-300gb
runs-on: ubuntu-latest
strategy:
matrix:
datasource: [mssql, mysql, postgres, mongodb, mariadb, oracle, none]
steps:
- name: Checkout repo
uses: actions/checkout@v4
@ -170,12 +173,19 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} &
docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} &
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} &
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} &
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} &
docker pull budibase/oracle-database:23.2-slim-faststart &
if [ "${{ matrix.datasource }}" == "mssql" ]; then
docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
elif [ "${{ matrix.datasource }}" == "mysql" ]; then
docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
elif [ "${{ matrix.datasource }}" == "postgres" ]; then
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
elif [ "${{ matrix.datasource }}" == "oracle" ]; then
docker pull budibase/oracle-database:23.2-slim-faststart
fi
docker pull minio/minio &
docker pull redis &
docker pull testcontainers/ryuk:0.5.1 &
@ -186,11 +196,19 @@ jobs:
- run: yarn --frozen-lockfile
- name: Test server
env:
DATASOURCE: ${{ matrix.datasource }}
run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then
node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }}
AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
if [ -n "$AFFECTED" ]; then
cd packages/server
if [ "${{ matrix.datasource }}" == "none" ]; then
yarn test --filter ./src/tests/filters/non-datasource-tests.js --passWithNoTests
else
yarn test --filter ./src/tests/filters/datasource-tests.js --passWithNoTests
fi
else
yarn test --scope=@budibase/server
echo "No affected tests to run"
fi
check-pro-submodule:

View File

@ -62,6 +62,7 @@ export default async function setup() {
},
])
.withLabels({ "com.budibase": "true" })
.withTmpFs({ "/data": "rw" })
.withReuse()
.withWaitStrategy(
Wait.forSuccessfulCommand(
@ -72,6 +73,7 @@ export default async function setup() {
const minio = new GenericContainer("minio/minio")
.withExposedPorts(9000)
.withCommand(["server", "/data"])
.withTmpFs({ "/data": "rw" })
.withEnvironment({
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",

@ -1 +1 @@
Subproject commit a56696a4af5667617746600fc75fe6a01744b692
Subproject commit bfeece324a03a3a5f25137bf3f8c66d5ed6103d8

View File

@ -1,12 +1,12 @@
#!/bin/bash
set -e
set -ex
if [[ -n $CI ]]
then
export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail "$@"
else
# --maxWorkers performs better in development
export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
jest --coverage --maxWorkers=2 --forceExit $@
jest --coverage --maxWorkers=2 --forceExit "$@"
fi

View File

@ -19,8 +19,7 @@ import {
} from "@budibase/types"
import {
DatabaseName,
getDatasource,
knexClient,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
import nock from "nock"
@ -69,7 +68,7 @@ describe("/datasources", () => {
{
status: 500,
body: {
message: "No datasource implementation found.",
message: 'No datasource implementation found called: "invalid"',
},
}
)
@ -163,21 +162,23 @@ describe("/datasources", () => {
})
})
})
})
describe.each([
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("%s", (_, dsProvider) => {
datasourceDescribe(
{ name: "%s", exclude: [DatabaseName.MONGODB, DatabaseName.SQS] },
({ config, dsProvider }) => {
let datasource: Datasource
let rawDatasource: Datasource
let client: Knex
beforeEach(async () => {
rawDatasource = await dsProvider
datasource = await config.api.datasource.create(rawDatasource)
client = await knexClient(rawDatasource)
const ds = await dsProvider()
rawDatasource = ds.rawDatasource!
datasource = ds.datasource!
client = ds.client!
jest.clearAllMocks()
nock.cleanAll()
})
describe("get", () => {
@ -491,5 +492,5 @@ describe("/datasources", () => {
)
})
})
})
})
}
)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -9,15 +9,20 @@ import {
import { automations } from "@budibase/pro"
import {
CreateRowActionRequest,
Datasource,
DocumentType,
PermissionLevel,
RowActionResponse,
Table,
TableRowActions,
} from "@budibase/types"
import * as setup from "./utilities"
import { generator, mocks } from "@budibase/backend-core/tests"
import { Expectations } from "../../../tests/utilities/api/base"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { generateRowActionsID } from "../../../db/utils"
const expectAutomationId = () =>
@ -969,70 +974,66 @@ describe("/rowsActions", () => {
status: 200,
})
})
it.each([
[
"internal",
async () => {
await config.newTenant()
await config.api.application.addSampleData(config.getAppId())
const tables = await config.api.table.fetch()
const table = tables.find(
t => t.sourceId === DEFAULT_BB_DATASOURCE_ID
)!
return table
},
],
[
"external",
async () => {
await config.newTenant()
const ds = await config.createDatasource({
datasource: await getDatasource(DatabaseName.POSTGRES),
})
const table = await config.api.table.save(
setup.structures.tableForDatasource(ds)
)
return table
},
],
])(
"should delete all the row actions (and automations) for its tables when a datasource is deleted",
async (_, getTable) => {
async function getRowActionsFromDb(tableId: string) {
return await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB()
const tableDoc = await db.tryGet<TableRowActions>(
generateRowActionsID(tableId)
)
return tableDoc
})
}
const table = await getTable()
const tableId = table._id!
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
const { actions } = (await getRowActionsFromDb(tableId))!
expect(Object.entries(actions)).toHaveLength(2)
const { automations } = await config.api.automation.fetch()
expect(automations).toHaveLength(2)
const datasource = await config.api.datasource.get(table.sourceId)
await config.api.datasource.delete(datasource)
const automationsResp = await config.api.automation.fetch()
expect(automationsResp.automations).toHaveLength(0)
expect(await getRowActionsFromDb(tableId)).toBeUndefined()
}
)
})
})
datasourceDescribe(
{ name: "row actions (%s)", only: [DatabaseName.SQS, DatabaseName.POSTGRES] },
({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource
})
async function getTable(): Promise<Table> {
if (isInternal) {
await config.api.application.addSampleData(config.getAppId())
const tables = await config.api.table.fetch()
return tables.find(t => t.sourceId === DEFAULT_BB_DATASOURCE_ID)!
} else {
const table = await config.api.table.save(
setup.structures.tableForDatasource(datasource!)
)
return table
}
}
it("should delete all the row actions (and automations) for its tables when a datasource is deleted", async () => {
async function getRowActionsFromDb(tableId: string) {
return await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB()
const tableDoc = await db.tryGet<TableRowActions>(
generateRowActionsID(tableId)
)
return tableDoc
})
}
const table = await getTable()
const tableId = table._id!
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
const { actions } = (await getRowActionsFromDb(tableId))!
expect(Object.entries(actions)).toHaveLength(2)
const { automations } = await config.api.automation.fetch()
expect(automations).toHaveLength(2)
const datasource = await config.api.datasource.get(table.sourceId)
await config.api.datasource.delete(datasource)
const automationsResp = await config.api.automation.fetch()
expect(automationsResp.automations).toHaveLength(0)
expect(await getRowActionsFromDb(tableId)).toBeUndefined()
})
}
)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,15 +1,15 @@
const setup = require("./utilities")
import { getConfig, afterAll as _afterAll, runStep } from "./utilities"
describe("test the bash action", () => {
let config = setup.getConfig()
let config = getConfig()
beforeAll(async () => {
await config.init()
})
afterAll(setup.afterAll)
afterAll(_afterAll)
it("should be able to execute a script", async () => {
let res = await setup.runStep("EXECUTE_BASH", {
let res = await runStep(config, "EXECUTE_BASH", {
code: "echo 'test'",
})
expect(res.stdout).toEqual("test\n")
@ -17,7 +17,7 @@ describe("test the bash action", () => {
})
it("should handle a null value", async () => {
let res = await setup.runStep("EXECUTE_BASH", {
let res = await runStep(config, "EXECUTE_BASH", {
code: null,
})
expect(res.stdout).toEqual(

View File

@ -31,7 +31,7 @@ describe("test the create row action", () => {
afterAll(setup.afterAll)
it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row,
})
expect(res.id).toBeDefined()
@ -43,7 +43,7 @@ describe("test the create row action", () => {
})
it("should return an error (not throw) when bad info provided", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: {
tableId: "invalid",
invalid: "invalid",
@ -53,7 +53,7 @@ describe("test the create row action", () => {
})
it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {})
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {})
expect(res.success).toEqual(false)
})
@ -76,7 +76,7 @@ describe("test the create row action", () => {
]
attachmentRow.file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow,
})
@ -111,7 +111,7 @@ describe("test the create row action", () => {
}
attachmentRow.single_file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow,
})
@ -146,7 +146,7 @@ describe("test the create row action", () => {
}
attachmentRow.single_file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow,
})

View File

@ -1,14 +1,20 @@
const setup = require("./utilities")
import { runStep, actions, getConfig } from "./utilities"
import { reset } from "timekeeper"
// need real Date for this test
const tk = require("timekeeper")
tk.reset()
reset()
describe("test the delay logic", () => {
const config = getConfig()
beforeAll(async () => {
await config.init()
})
it("should be able to run the delay", async () => {
const time = 100
const before = Date.now()
await setup.runStep(setup.actions.DELAY.stepId, { time: time })
await runStep(config, actions.DELAY.stepId, { time: time })
const now = Date.now()
// divide by two just so that test will always pass as long as there was some sort of delay
expect(now - before).toBeGreaterThanOrEqual(time / 2)

View File

@ -1,4 +1,4 @@
const setup = require("./utilities")
import * as setup from "./utilities"
describe("test the delete row action", () => {
let table: any
@ -20,32 +20,29 @@ describe("test the delete row action", () => {
afterAll(setup.afterAll)
it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.DELETE_ROW.stepId,
inputs
)
expect(res.success).toEqual(true)
expect(res.response).toBeDefined()
expect(res.row._id).toEqual(row._id)
let error
try {
await config.getRow(table._id, res.row._id)
} catch (err) {
error = err
}
expect(error).toBeDefined()
})
it("check usage quota attempts", async () => {
await setup.runInProd(async () => {
await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
await setup.runStep(config, setup.actions.DELETE_ROW.stepId, inputs)
})
})
it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, {})
const res = await setup.runStep(config, setup.actions.DELETE_ROW.stepId, {})
expect(res.success).toEqual(false)
})
it("should return an error when table doesn't exist", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.DELETE_ROW.stepId, {
tableId: "invalid",
id: "invalid",
revision: "invalid",

View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.discord.stepId, {
const res = await runStep(config, actions.discord.stepId, {
url: "http://www.example.com",
username: "joe_bloggs",
})

View File

@ -1,65 +1,77 @@
import { Datasource, Query } from "@budibase/types"
import * as setup from "./utilities"
import { DatabaseName } from "../../integrations/tests/utils"
import {
DatabaseName,
datasourceDescribe,
} from "../../integrations/tests/utils"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
describe.each([
DatabaseName.POSTGRES,
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
])("execute query action (%s)", name => {
let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
const config = setup.getConfig()
datasourceDescribe(
{
name: "execute query action",
exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
},
({ config, dsProvider }) => {
let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
beforeAll(async () => {
await config.init()
const testSetup = await setup.setupTestDatasource(config, name)
datasource = testSetup.datasource
client = testSetup.client
})
beforeEach(async () => {
tableName = await setup.createTestTable(client, {
a: { type: "string" },
b: { type: "number" },
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
await setup.insertTestData(client, tableName, [{ a: "string", b: 1 }])
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
afterEach(async () => {
await client.schema.dropTable(tableName)
})
afterAll(setup.afterAll)
it("should be able to execute a query", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: { queryId: query._id },
beforeEach(async () => {
tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.string("a")
table.integer("b")
})
await client(tableName).insert({ a: "string", b: 1 })
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(true)
})
it("should handle a null query value", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: null,
afterEach(async () => {
await client.schema.dropTable(tableName)
})
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false)
})
it("should handle an error executing a query", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: { queryId: "wrong_id" },
it("should be able to execute a query", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: query._id },
}
)
expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(true)
})
expect(res.response).toBeDefined()
expect(res.success).toEqual(false)
})
})
it("should handle a null query value", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: null,
}
)
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false)
})
it("should handle an error executing a query", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: "wrong_id" },
}
)
expect(res.response).toBeDefined()
expect(res.success).toEqual(false)
})
}
)

View File

@ -1,15 +1,15 @@
const setup = require("./utilities")
import { getConfig, afterAll as _afterAll, runStep, actions } from "./utilities"
describe("test the execute script action", () => {
let config = setup.getConfig()
let config = getConfig()
beforeAll(async () => {
await config.init()
})
afterAll(setup.afterAll)
afterAll(_afterAll)
it("should be able to execute a script", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: "return 1 + 1",
})
expect(res.value).toEqual(2)
@ -17,7 +17,7 @@ describe("test the execute script action", () => {
})
it("should handle a null value", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: null,
})
expect(res.response.message).toEqual("Invalid inputs")
@ -25,8 +25,9 @@ describe("test the execute script action", () => {
})
it("should be able to get a value from context", async () => {
const res = await setup.runStep(
setup.actions.EXECUTE_SCRIPT.stepId,
const res = await runStep(
config,
actions.EXECUTE_SCRIPT.stepId,
{
code: "return steps.map(d => d.value)",
},
@ -40,7 +41,7 @@ describe("test the execute script action", () => {
})
it("should be able to handle an error gracefully", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: "return something.map(x => x.name)",
})
expect(res.response).toEqual("ReferenceError: something is not defined")

View File

@ -2,13 +2,19 @@ import * as setup from "./utilities"
import { FilterConditions } from "../steps/filter"
describe("test the filter logic", () => {
const config = setup.getConfig()
beforeAll(async () => {
await config.init()
})
async function checkFilter(
field: any,
condition: string,
value: any,
pass = true
) {
let res = await setup.runStep(setup.actions.FILTER.stepId, {
let res = await setup.runStep(config, setup.actions.FILTER.stepId, {
field,
condition,
value,

View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.integromat.stepId, {
const res = await runStep(config, actions.integromat.stepId, {
url: "http://www.example.com",
})
expect(res.response.foo).toEqual("bar")
@ -38,7 +38,7 @@ describe("test the outgoing webhook action", () => {
.post("/", payload)
.reply(200, { foo: "bar" })
const res = await runStep(actions.integromat.stepId, {
const res = await runStep(config, actions.integromat.stepId, {
body: { value: JSON.stringify(payload) },
url: "http://www.example.com",
})
@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
})
it("should return a 400 if the JSON payload string is malformed", async () => {
const res = await runStep(actions.integromat.stepId, {
const res = await runStep(config, actions.integromat.stepId, {
body: { value: "{ invalid json }" },
url: "http://www.example.com",
})

View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action and default to 'get'", async () => {
nock("http://www.example.com/").get("/").reply(200, { foo: "bar" })
const res = await runStep(actions.n8n.stepId, {
const res = await runStep(config, actions.n8n.stepId, {
url: "http://www.example.com",
body: {
test: "IGNORE_ME",
@ -30,7 +30,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com/")
.post("/", { name: "Adam", age: 9 })
.reply(200)
const res = await runStep(actions.n8n.stepId, {
const res = await runStep(config, actions.n8n.stepId, {
body: {
value: JSON.stringify({ name: "Adam", age: 9 }),
},
@ -42,7 +42,7 @@ describe("test the outgoing webhook action", () => {
it("should return a 400 if the JSON payload string is malformed", async () => {
const payload = `{ value1 1 }`
const res = await runStep(actions.n8n.stepId, {
const res = await runStep(config, actions.n8n.stepId, {
value1: "ONE",
body: {
value: payload,
@ -59,7 +59,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com/")
.head("/", body => body === "")
.reply(200)
const res = await runStep(actions.n8n.stepId, {
const res = await runStep(config, actions.n8n.stepId, {
url: "http://www.example.com",
method: "HEAD",
body: {

View File

@ -62,13 +62,13 @@ describe("test the openai action", () => {
afterAll(_afterAll)
it("should be able to receive a response from ChatGPT given a prompt", async () => {
const res = await runStep("OPENAI", { prompt: OPENAI_PROMPT })
const res = await runStep(config, "OPENAI", { prompt: OPENAI_PROMPT })
expect(res.response).toEqual("This is a test")
expect(res.success).toBeTruthy()
})
it("should present the correct error message when a prompt is not provided", async () => {
const res = await runStep("OPENAI", { prompt: null })
const res = await runStep(config, "OPENAI", { prompt: null })
expect(res.response).toEqual(
"Budibase OpenAI Automation Failed: No prompt supplied"
)
@ -91,7 +91,7 @@ describe("test the openai action", () => {
} as any)
)
const res = await runStep("OPENAI", {
const res = await runStep(config, "OPENAI", {
prompt: OPENAI_PROMPT,
})
@ -106,7 +106,7 @@ describe("test the openai action", () => {
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?"
await runStep("OPENAI", {
await runStep(config, "OPENAI", {
model: "gpt-4o-mini",
prompt,
})

View File

@ -18,7 +18,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com")
.post("/", { a: 1 })
.reply(200, { foo: "bar" })
const res = await runStep(actions.OUTGOING_WEBHOOK.stepId, {
const res = await runStep(config, actions.OUTGOING_WEBHOOK.stepId, {
requestMethod: "POST",
url: "www.example.com",
requestBody: JSON.stringify({ a: 1 }),
@ -28,7 +28,7 @@ describe("test the outgoing webhook action", () => {
})
it("should return an error if something goes wrong in fetch", async () => {
const res = await runStep(actions.OUTGOING_WEBHOOK.stepId, {
const res = await runStep(config, actions.OUTGOING_WEBHOOK.stepId, {
requestMethod: "GET",
url: "www.invalid.com",
})

View File

@ -33,7 +33,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending",
limit: 10,
}
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)
@ -48,7 +52,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending",
limit: 10,
}
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)
@ -65,7 +73,11 @@ describe("Test a query step automation", () => {
limit: 10,
onEmptyFilter: "none",
}
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(false)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0)
@ -85,7 +97,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending",
limit: 10,
}
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(false)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0)
@ -100,7 +116,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending",
limit: 10,
}
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)

View File

@ -1,9 +1,14 @@
import * as automation from "../../index"
import * as setup from "../utilities"
import { LoopStepType, FieldType, Table } from "@budibase/types"
import { LoopStepType, FieldType, Table, Datasource } from "@budibase/types"
import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
import { DatabaseName } from "../../../integrations/tests/utils"
import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { FilterConditions } from "../../../automations/steps/filter"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
describe("Automation Scenarios", () => {
let config = setup.getConfig()
@ -107,96 +112,6 @@ describe("Automation Scenarios", () => {
expect(results.steps[2].outputs.rows).toHaveLength(1)
})
it("should query an external database for some data then insert than into an internal table", async () => {
const { datasource, client } = await setup.setupTestDatasource(
config,
DatabaseName.MYSQL
)
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = await setup.createTestTable(client, {
name: { type: "string" },
age: { type: "number" },
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await setup.insertTestData(client, tableName, rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
)
})
})
it("should trigger an automation which creates and then updates a row", async () => {
const table = await config.createTable({
name: "TestTable",
@ -517,3 +432,104 @@ describe("Automation Scenarios", () => {
expect(results.steps[0].outputs.message).toContain("example.com")
})
})
datasourceDescribe(
{ name: "", only: [DatabaseName.MYSQL] },
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
it("should query an external database for some data then insert than into an internal table", async () => {
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.string("name")
table.integer("age")
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await client(tableName).insert(rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
config,
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
)
})
})
}
)

View File

@ -18,7 +18,7 @@ function generateResponse(to: string, from: string) {
}
}
const setup = require("./utilities")
import * as setup from "./utilities"
describe("test the outgoing webhook action", () => {
let inputs
@ -58,6 +58,7 @@ describe("test the outgoing webhook action", () => {
}
let resp = generateResponse(inputs.to, inputs.from)
const res = await setup.runStep(
config,
setup.actions.SEND_EMAIL_SMTP.stepId,
inputs
)

View File

@ -1,8 +1,8 @@
const setup = require("./utilities")
import { getConfig, afterAll as _afterAll, runStep, actions } from "./utilities"
describe("test the server log action", () => {
let config = setup.getConfig()
let inputs
let config = getConfig()
let inputs: any
beforeAll(async () => {
await config.init()
@ -10,10 +10,10 @@ describe("test the server log action", () => {
text: "log message",
}
})
afterAll(setup.afterAll)
afterAll(_afterAll)
it("should be able to log the text", async () => {
let res = await setup.runStep(setup.actions.SERVER_LOG.stepId, inputs)
let res = await runStep(config, actions.SERVER_LOG.stepId, inputs)
expect(res.message).toEqual(`App ${config.getAppId()} - ${inputs.text}`)
expect(res.success).toEqual(true)
})

View File

@ -29,6 +29,7 @@ describe("Test triggering an automation from another automation", () => {
},
}
const res = await setup.runStep(
config,
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs
)
@ -44,6 +45,7 @@ describe("Test triggering an automation from another automation", () => {
},
}
const res = await setup.runStep(
config,
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs
)

View File

@ -34,7 +34,11 @@ describe("test the update row action", () => {
afterAll(setup.afterAll)
it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, inputs)
const res = await setup.runStep(
config,
setup.actions.UPDATE_ROW.stepId,
inputs
)
expect(res.success).toEqual(true)
const updatedRow = await config.api.row.get(table._id!, res.id)
expect(updatedRow.name).toEqual("Updated name")
@ -42,12 +46,12 @@ describe("test the update row action", () => {
})
it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {})
const res = await setup.runStep(config, setup.actions.UPDATE_ROW.stepId, {})
expect(res.success).toEqual(false)
})
it("should return an error when table doesn't exist", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {
const res = await setup.runStep(config, setup.actions.UPDATE_ROW.stepId, {
row: { _id: "invalid" },
rowId: "invalid",
})
@ -90,16 +94,20 @@ describe("test the update row action", () => {
expect(getResp.user1[0]._id).toEqual(user1._id)
expect(getResp.user2[0]._id).toEqual(user2._id)
let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {
rowId: row._id,
row: {
_id: row._id,
_rev: row._rev,
tableId: row.tableId,
user1: [user2._id],
user2: "",
},
})
let stepResp = await setup.runStep(
config,
setup.actions.UPDATE_ROW.stepId,
{
rowId: row._id,
row: {
_id: row._id,
_rev: row._rev,
tableId: row.tableId,
user1: [user2._id],
user2: "",
},
}
)
expect(stepResp.success).toEqual(true)
getResp = await config.api.row.get(table._id!, row._id!)
@ -143,23 +151,27 @@ describe("test the update row action", () => {
expect(getResp.user1[0]._id).toEqual(user1._id)
expect(getResp.user2[0]._id).toEqual(user2._id)
let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {
rowId: row._id,
row: {
_id: row._id,
_rev: row._rev,
tableId: row.tableId,
user1: [user2._id],
user2: "",
},
meta: {
fields: {
user2: {
clearRelationships: true,
let stepResp = await setup.runStep(
config,
setup.actions.UPDATE_ROW.stepId,
{
rowId: row._id,
row: {
_id: row._id,
_rev: row._rev,
tableId: row.tableId,
user1: [user2._id],
user2: "",
},
meta: {
fields: {
user2: {
clearRelationships: true,
},
},
},
},
})
}
)
expect(stepResp.success).toEqual(true)
getResp = await config.api.row.get(table._id!, row._id!)

View File

@ -1,22 +1,16 @@
import TestConfig from "../../../tests/utilities/TestConfiguration"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import { context } from "@budibase/backend-core"
import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
import emitter from "../../../events/index"
import env from "../../../environment"
import { AutomationActionStepId, Datasource } from "@budibase/types"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
import {
getDatasource,
knexClient,
DatabaseName,
} from "../../../integrations/tests/utils"
let config: TestConfig
let config: TestConfiguration
export function getConfig(): TestConfig {
export function getConfig(): TestConfiguration {
if (!config) {
config = new TestConfig(true)
config = new TestConfiguration(true)
}
return config
}
@ -39,7 +33,12 @@ export async function runInProd(fn: any) {
}
}
export async function runStep(stepId: string, inputs: any, stepContext?: any) {
export async function runStep(
config: TestConfiguration,
stepId: string,
inputs: any,
stepContext?: any
) {
async function run() {
let step = await getAction(stepId as AutomationActionStepId)
expect(step).toBeDefined()
@ -55,7 +54,7 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
emitter,
})
}
if (config?.appId) {
if (config.appId) {
return context.doInContext(config?.appId, async () => {
return run()
})
@ -64,31 +63,8 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
}
}
export async function createTestTable(client: Knex, schema: any) {
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
for (const fieldName in schema) {
const field = schema[fieldName]
if (field.type === "string") {
table.string(fieldName)
} else if (field.type === "number") {
table.integer(fieldName)
}
}
})
return tableName
}
export async function insertTestData(
client: Knex,
tableName: string,
rows: any[]
) {
await client(tableName).insert(rows)
}
export async function saveTestQuery(
config: TestConfig,
config: TestConfiguration,
client: Knex,
tableName: string,
datasource: Datasource
@ -107,15 +83,5 @@ export async function saveTestQuery(
})
}
export async function setupTestDatasource(
config: TestConfig,
dbName: DatabaseName
) {
const db = await getDatasource(dbName)
const datasource = await config.api.datasource.create(db)
const client = await knexClient(db)
return { datasource, client }
}
export const apiKey = "test"
export const actions = BUILTIN_ACTION_DEFINITIONS

View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.zapier.stepId, {
const res = await runStep(config, actions.zapier.stepId, {
url: "http://www.example.com",
})
expect(res.response.foo).toEqual("bar")
@ -38,7 +38,7 @@ describe("test the outgoing webhook action", () => {
.post("/", { ...payload, platform: "budibase" })
.reply(200, { foo: "bar" })
const res = await runStep(actions.zapier.stepId, {
const res = await runStep(config, actions.zapier.stepId, {
body: { value: JSON.stringify(payload) },
url: "http://www.example.com",
})
@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
})
it("should return a 400 if the JSON payload string is malformed", async () => {
const res = await runStep(actions.zapier.stepId, {
const res = await runStep(config, actions.zapier.stepId, {
body: { value: "{ invalid json }" },
url: "http://www.example.com",
})

View File

@ -1,10 +1,5 @@
import * as setup from "../api/routes/tests/utilities"
import { Datasource, FieldType } from "@budibase/types"
import {
DatabaseName,
getDatasource,
knexClient,
} from "../integrations/tests/utils"
import { DatabaseName, datasourceDescribe } from "../integrations/tests/utils"
import { generator } from "@budibase/backend-core/tests"
import { Knex } from "knex"
@ -15,31 +10,24 @@ function uniqueTableName(length?: number): string {
.substring(0, length || 10)
}
const config = setup.getConfig()!
describe("mysql integrations", () => {
let datasource: Datasource
let client: Knex
beforeAll(async () => {
await config.init()
const rawDatasource = await getDatasource(DatabaseName.MYSQL)
datasource = await config.api.datasource.create(rawDatasource)
client = await knexClient(rawDatasource)
})
afterAll(config.end)
describe("Integration compatibility with mysql search_path", () => {
let datasource: Datasource
datasourceDescribe(
{
name: "Integration compatibility with mysql search_path",
only: [DatabaseName.MYSQL],
},
({ config, dsProvider }) => {
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
const database = generator.guid()
const database2 = generator.guid()
beforeAll(async () => {
rawDatasource = await getDatasource(DatabaseName.MYSQL)
client = await knexClient(rawDatasource)
const ds = await dsProvider()
rawDatasource = ds.rawDatasource!
datasource = ds.datasource!
client = ds.client!
await client.raw(`CREATE DATABASE \`${database}\`;`)
await client.raw(`CREATE DATABASE \`${database2}\`;`)
@ -87,11 +75,25 @@ describe("mysql integrations", () => {
const schema = res.datasource.entities![repeated_table_name].schema
expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
})
})
}
)
datasourceDescribe(
{
name: "POST /api/datasources/:datasourceId/schema",
only: [DatabaseName.MYSQL],
},
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
describe("POST /api/datasources/:datasourceId/schema", () => {
let tableName: string
beforeEach(async () => {
tableName = uniqueTableName()
})
@ -122,5 +124,5 @@ describe("mysql integrations", () => {
expect(table).toBeDefined()
expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
})
})
})
}
)

View File

@ -1,105 +1,230 @@
import * as setup from "../api/routes/tests/utilities"
import { Datasource, FieldType, Table } from "@budibase/types"
import _ from "lodash"
import { generator } from "@budibase/backend-core/tests"
import {
DatabaseName,
getDatasource,
datasourceDescribe,
knexClient,
} from "../integrations/tests/utils"
import { Knex } from "knex"
const config = setup.getConfig()!
datasourceDescribe(
{ name: "postgres integrations", only: [DatabaseName.POSTGRES] },
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
describe("postgres integrations", () => {
let datasource: Datasource
let client: Knex
beforeAll(async () => {
await config.init()
const rawDatasource = await getDatasource(DatabaseName.POSTGRES)
datasource = await config.api.datasource.create(rawDatasource)
client = await knexClient(rawDatasource)
})
afterAll(config.end)
describe("POST /api/datasources/:datasourceId/schema", () => {
let tableName: string
beforeEach(async () => {
tableName = generator.guid().replaceAll("-", "").substring(0, 10)
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
afterEach(async () => {
await client.schema.dropTableIfExists(tableName)
})
afterAll(config.end)
it("recognises when a table has no primary key", async () => {
await client.schema.createTable(tableName, table => {
table.increments("id", { primaryKey: false })
describe("POST /api/datasources/:datasourceId/schema", () => {
let tableName: string
beforeEach(async () => {
tableName = generator.guid().replaceAll("-", "").substring(0, 10)
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
afterEach(async () => {
await client.schema.dropTableIfExists(tableName)
})
expect(response.errors).toEqual({
[tableName]: "Table must have a primary key.",
})
})
it("recognises when a table has no primary key", async () => {
await client.schema.createTable(tableName, table => {
table.increments("id", { primaryKey: false })
})
it("recognises when a table is using a reserved column name", async () => {
await client.schema.createTable(tableName, table => {
table.increments("_id").primary()
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(response.errors).toEqual({
[tableName]: "Table contains invalid columns.",
})
})
it("recognises enum columns as options", async () => {
const tableName = `orders_${generator
.guid()
.replaceAll("-", "")
.substring(0, 6)}`
await client.schema.createTable(tableName, table => {
table.increments("order_id").primary()
table.string("customer_name").notNullable()
table.enum("status", ["pending", "processing", "shipped"], {
useNative: true,
enumName: `${tableName}_status`,
expect(response.errors).toEqual({
[tableName]: "Table must have a primary key.",
})
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
it("recognises when a table is using a reserved column name", async () => {
await client.schema.createTable(tableName, table => {
table.increments("_id").primary()
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(response.errors).toEqual({
[tableName]: "Table contains invalid columns.",
})
})
const table = response.datasource.entities?.[tableName]
it("recognises enum columns as options", async () => {
const tableName = `orders_${generator
.guid()
.replaceAll("-", "")
.substring(0, 6)}`
expect(table).toBeDefined()
expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS)
await client.schema.createTable(tableName, table => {
table.increments("order_id").primary()
table.string("customer_name").notNullable()
table.enum("status", ["pending", "processing", "shipped"], {
useNative: true,
enumName: `${tableName}_status`,
})
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const table = response.datasource.entities?.[tableName]
expect(table).toBeDefined()
expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS)
})
})
})
describe("Integration compatibility with postgres search_path", () => {
describe("check custom column types", () => {
beforeAll(async () => {
await client.schema.createTable("binaryTable", table => {
table.binary("id").primary()
table.string("column1")
table.integer("column2")
})
})
it("should handle binary columns", async () => {
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(response.datasource.entities).toBeDefined()
const table = response.datasource.entities?.["binaryTable"]
expect(table).toBeDefined()
expect(table?.schema.id.externalType).toBe("bytea")
const row = await config.api.row.save(table?._id!, {
id: "1111",
column1: "hello",
column2: 222,
})
expect(row._id).toBeDefined()
const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
expect(JSON.parse(decoded)[0]).toBe("1111")
})
})
describe("check fetching null/not null table", () => {
beforeAll(async () => {
await client.schema.createTable("nullableTable", table => {
table.increments("order_id").primary()
table.integer("order_number").notNullable()
})
})
it("should be able to change the table to allow nullable and refetch this", async () => {
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entities = response.datasource.entities
expect(entities).toBeDefined()
const nullableTable = entities?.["nullableTable"]
expect(nullableTable).toBeDefined()
expect(
nullableTable?.schema["order_number"].constraints?.presence
).toEqual(true)
// need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase
// is aware of - therefore we can try to fetch and make sure BB updates correctly
await client.schema.alterTable("nullableTable", table => {
table.setNullable("order_number")
})
const responseAfter = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entitiesAfter = responseAfter.datasource.entities
expect(entitiesAfter).toBeDefined()
const nullableTableAfter = entitiesAfter?.["nullableTable"]
expect(nullableTableAfter).toBeDefined()
expect(
nullableTableAfter?.schema["order_number"].constraints?.presence
).toBeUndefined()
})
})
describe("money field 💰", () => {
const tableName = "moneytable"
let table: Table
beforeAll(async () => {
await client.raw(`
CREATE TABLE ${tableName} (
id serial PRIMARY KEY,
price money
)
`)
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
table = response.datasource.entities![tableName]
})
it("should be able to import a money field", async () => {
expect(table).toBeDefined()
expect(table?.schema.price.type).toBe(FieldType.NUMBER)
})
it("should be able to search a money field", async () => {
await config.api.row.bulkImport(table._id!, {
rows: [{ price: 200 }, { price: 300 }],
})
const { rows } = await config.api.row.search(table._id!, {
query: {
equal: {
price: 200,
},
},
})
expect(rows).toHaveLength(1)
expect(rows[0].price).toBe("200.00")
})
it("should be able to update a money field", async () => {
let row = await config.api.row.save(table._id!, { price: 200 })
expect(row.price).toBe("200.00")
row = await config.api.row.save(table._id!, { ...row, price: 300 })
expect(row.price).toBe("300.00")
row = await config.api.row.save(table._id!, { ...row, price: "400.00" })
expect(row.price).toBe("400.00")
})
})
}
)
datasourceDescribe(
{
name: "Integration compatibility with postgres search_path",
only: [DatabaseName.POSTGRES],
},
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
let schema1: string
let schema2: string
beforeEach(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
const rawDatasource = ds.rawDatasource!
schema1 = generator.guid().replaceAll("-", "")
schema2 = generator.guid().replaceAll("-", "")
const rawDatasource = await getDatasource(DatabaseName.POSTGRES)
client = await knexClient(rawDatasource)
await client.schema.createSchema(schema1)
@ -161,122 +286,5 @@ describe("postgres integrations", () => {
const schema = response.datasource.entities?.[repeated_table_name].schema
expect(Object.keys(schema || {}).sort()).toEqual(["id", "val1"])
})
})
describe("check custom column types", () => {
beforeAll(async () => {
await client.schema.createTable("binaryTable", table => {
table.binary("id").primary()
table.string("column1")
table.integer("column2")
})
})
it("should handle binary columns", async () => {
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(response.datasource.entities).toBeDefined()
const table = response.datasource.entities?.["binaryTable"]
expect(table).toBeDefined()
expect(table?.schema.id.externalType).toBe("bytea")
const row = await config.api.row.save(table?._id!, {
id: "1111",
column1: "hello",
column2: 222,
})
expect(row._id).toBeDefined()
const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
expect(JSON.parse(decoded)[0]).toBe("1111")
})
})
describe("check fetching null/not null table", () => {
beforeAll(async () => {
await client.schema.createTable("nullableTable", table => {
table.increments("order_id").primary()
table.integer("order_number").notNullable()
})
})
it("should be able to change the table to allow nullable and refetch this", async () => {
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entities = response.datasource.entities
expect(entities).toBeDefined()
const nullableTable = entities?.["nullableTable"]
expect(nullableTable).toBeDefined()
expect(
nullableTable?.schema["order_number"].constraints?.presence
).toEqual(true)
// need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase
// is aware of - therefore we can try to fetch and make sure BB updates correctly
await client.schema.alterTable("nullableTable", table => {
table.setNullable("order_number")
})
const responseAfter = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entitiesAfter = responseAfter.datasource.entities
expect(entitiesAfter).toBeDefined()
const nullableTableAfter = entitiesAfter?.["nullableTable"]
expect(nullableTableAfter).toBeDefined()
expect(
nullableTableAfter?.schema["order_number"].constraints?.presence
).toBeUndefined()
})
})
describe("money field 💰", () => {
const tableName = "moneytable"
let table: Table
beforeAll(async () => {
await client.raw(`
CREATE TABLE ${tableName} (
id serial PRIMARY KEY,
price money
)
`)
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
table = response.datasource.entities![tableName]
})
it("should be able to import a money field", async () => {
expect(table).toBeDefined()
expect(table?.schema.price.type).toBe(FieldType.NUMBER)
})
it("should be able to search a money field", async () => {
await config.api.row.bulkImport(table._id!, {
rows: [{ price: 200 }, { price: 300 }],
})
const { rows } = await config.api.row.search(table._id!, {
query: {
equal: {
price: 200,
},
},
})
expect(rows).toHaveLength(1)
expect(rows[0].price).toBe("200.00")
})
it("should be able to update a money field", async () => {
let row = await config.api.row.save(table._id!, { price: 200 })
expect(row.price).toBe("200.00")
row = await config.api.row.save(table._id!, { ...row, price: 300 })
expect(row.price).toBe("300.00")
row = await config.api.row.save(table._id!, { ...row, price: "400.00" })
expect(row.price).toBe("400.00")
})
})
})
}
)

View File

@ -120,7 +120,7 @@ export async function getIntegration(integration: SourceName) {
}
}
}
throw new Error("No datasource implementation found.")
throw new Error(`No datasource implementation found called: "${integration}"`)
}
export default {

View File

@ -7,8 +7,10 @@ import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import * as oracle from "./oracle"
import { testContainerUtils } from "@budibase/backend-core/tests"
import { Knex } from "knex"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
export type DatasourceProvider = () => Promise<Datasource>
export type DatasourceProvider = () => Promise<Datasource | undefined>
export const { startContainer } = testContainerUtils
@ -19,6 +21,7 @@ export enum DatabaseName {
SQL_SERVER = "mssql",
MARIADB = "mariadb",
ORACLE = "oracle",
SQS = "sqs",
}
const providers: Record<DatabaseName, DatasourceProvider> = {
@ -28,30 +31,143 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
[DatabaseName.SQL_SERVER]: mssql.getDatasource,
[DatabaseName.MARIADB]: mariadb.getDatasource,
[DatabaseName.ORACLE]: oracle.getDatasource,
[DatabaseName.SQS]: async () => undefined,
}
export function getDatasourceProviders(
...sourceNames: DatabaseName[]
): Promise<Datasource>[] {
return sourceNames.map(sourceName => providers[sourceName]())
export interface DatasourceDescribeOpts {
name: string
only?: DatabaseName[]
exclude?: DatabaseName[]
}
export function getDatasourceProvider(
export interface DatasourceDescribeReturnPromise {
rawDatasource: Datasource | undefined
datasource: Datasource | undefined
client: Knex | undefined
}
export interface DatasourceDescribeReturn {
name: DatabaseName
config: TestConfiguration
dsProvider: () => Promise<DatasourceDescribeReturnPromise>
isInternal: boolean
isExternal: boolean
isSql: boolean
isMySQL: boolean
isPostgres: boolean
isMongodb: boolean
isMSSQL: boolean
isOracle: boolean
}
async function createDatasources(
config: TestConfiguration,
name: DatabaseName
): Promise<DatasourceDescribeReturnPromise> {
await config.init()
const rawDatasource = await getDatasource(name)
let datasource: Datasource | undefined
if (rawDatasource) {
datasource = await config.api.datasource.create(rawDatasource)
}
let client: Knex | undefined
if (rawDatasource) {
try {
client = await knexClient(rawDatasource)
} catch (e) {
// ignore
}
}
return {
rawDatasource,
datasource,
client,
}
}
// Jest doesn't allow test files to exist with no tests in them. When we run
// these tests in CI, we break them out by data source, and there are a bunch of
// test files that only run for a subset of data sources, and for the rest of
// them they will be empty test files. Defining a dummy test makes it so that
// Jest doesn't error in this situation.
function createDummyTest() {
describe("no tests", () => {
it("no tests", () => {
// no tests
})
})
}
export function datasourceDescribe(
opts: DatasourceDescribeOpts,
cb: (args: DatasourceDescribeReturn) => void
) {
if (process.env.DATASOURCE === "none") {
createDummyTest()
return
}
const { name, only, exclude } = opts
if (only && exclude) {
throw new Error("you can only supply one of 'only' or 'exclude'")
}
let databases = Object.values(DatabaseName)
if (only) {
databases = only
} else if (exclude) {
databases = databases.filter(db => !exclude.includes(db))
}
if (process.env.DATASOURCE) {
databases = databases.filter(db => db === process.env.DATASOURCE)
}
if (databases.length === 0) {
createDummyTest()
return
}
describe.each(databases)(name, name => {
const config = new TestConfiguration()
afterAll(() => {
config.end()
})
cb({
name,
config,
dsProvider: () => createDatasources(config, name),
isInternal: name === DatabaseName.SQS,
isExternal: name !== DatabaseName.SQS,
isSql: [
DatabaseName.MARIADB,
DatabaseName.MYSQL,
DatabaseName.POSTGRES,
DatabaseName.SQL_SERVER,
DatabaseName.ORACLE,
].includes(name),
isMySQL: name === DatabaseName.MYSQL,
isPostgres: name === DatabaseName.POSTGRES,
isMongodb: name === DatabaseName.MONGODB,
isMSSQL: name === DatabaseName.SQL_SERVER,
isOracle: name === DatabaseName.ORACLE,
})
})
}
function getDatasource(
sourceName: DatabaseName
): DatasourceProvider {
return providers[sourceName]
}
export function getDatasource(sourceName: DatabaseName): Promise<Datasource> {
): Promise<Datasource | undefined> {
return providers[sourceName]()
}
export async function getDatasources(
...sourceNames: DatabaseName[]
): Promise<Datasource[]> {
return Promise.all(sourceNames.map(sourceName => providers[sourceName]()))
}
export async function knexClient(ds: Datasource) {
switch (ds.source) {
case SourceName.POSTGRES: {

View File

@ -31,7 +31,7 @@ export async function getDatasource(): Promise<Datasource> {
new GenericContainer(MARIADB_IMAGE)
.withExposedPorts(3306)
.withEnvironment({ MARIADB_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MariaDBWaitStrategy())
.withWaitStrategy(new MariaDBWaitStrategy().withStartupTimeout(20000))
)
}

View File

@ -18,7 +18,7 @@ export async function getDatasource(): Promise<Datasource> {
.withWaitStrategy(
Wait.forSuccessfulCommand(
`mongosh --eval "db.version()"`
).withStartupTimeout(10000)
).withStartupTimeout(20000)
)
)
}

View File

@ -24,7 +24,7 @@ export async function getDatasource(): Promise<Datasource> {
.withWaitStrategy(
Wait.forSuccessfulCommand(
"/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Password_123 -q 'SELECT 1'"
)
).withStartupTimeout(20000)
)
)
}

View File

@ -34,7 +34,7 @@ export async function getDatasource(): Promise<Datasource> {
new GenericContainer(MYSQL_IMAGE)
.withExposedPorts(3306)
.withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
.withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
.withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(20000))
)
}

View File

@ -23,7 +23,11 @@ export async function getDatasource(): Promise<Datasource> {
.withEnvironment({
ORACLE_PASSWORD: password,
})
.withWaitStrategy(Wait.forLogMessage("DATABASE IS READY TO USE!"))
.withWaitStrategy(
Wait.forLogMessage("DATABASE IS READY TO USE!").withStartupTimeout(
20000
)
)
)
}

View File

@ -16,7 +16,7 @@ export async function getDatasource(): Promise<Datasource> {
.withWaitStrategy(
Wait.forSuccessfulCommand(
"pg_isready -h localhost -p 5432"
).withStartupTimeout(10000)
).withStartupTimeout(20000)
)
)
}

View File

@ -7,226 +7,214 @@ import {
Table,
} from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import {
DatabaseName,
getDatasource,
datasourceDescribe,
} from "../../../../../integrations/tests/utils"
import { tableForDatasource } from "../../../../../tests/utilities/structures"
// These test cases are only for things that cannot be tested through the API
// (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead.
describe.each([
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("search sdk (%s)", (name, dsProvider) => {
const isInternal = name === "internal"
const config = new TestConfiguration()
datasourceDescribe(
{ name: "search sdk (%s)", exclude: [DatabaseName.MONGODB] },
({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
let table: Table
let datasource: Datasource | undefined
let table: Table
beforeAll(async () => {
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
datasource: await dsProvider,
})
}
})
beforeEach(async () => {
const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata =
isInternal
? {
name: "id",
type: FieldType.AUTO,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
}
: {
name: "id",
type: FieldType.NUMBER,
autocolumn: true,
}
table = await config.api.table.save(
tableForDatasource(datasource, {
primary: ["id"],
schema: {
id: idFieldSchema,
name: {
name: "name",
type: FieldType.STRING,
},
surname: {
name: "surname",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
address: {
name: "address",
type: FieldType.STRING,
},
},
})
)
for (let i = 0; i < 10; i++) {
await config.api.row.save(table._id!, {
name: generator.first(),
surname: generator.last(),
age: generator.age(),
address: generator.address(),
})
}
})
afterAll(async () => {
config.end()
})
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(rows).toHaveLength(10)
for (const row of rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource
})
})
!isInternal &&
it("will decode _id in oneOf query", async () => {
await config.doInContext(config.appId, async () => {
const result = await search({
tableId: table._id!,
query: {
oneOf: {
_id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
beforeEach(async () => {
const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata =
isInternal
? {
name: "id",
type: FieldType.AUTO,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
}
: {
name: "id",
type: FieldType.NUMBER,
autocolumn: true,
}
table = await config.api.table.save(
tableForDatasource(datasource, {
primary: ["id"],
schema: {
id: idFieldSchema,
name: {
name: "name",
type: FieldType.STRING,
},
surname: {
name: "surname",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
address: {
name: "address",
type: FieldType.STRING,
},
},
})
)
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual(
expect.arrayContaining([1, 4, 8])
)
})
})
it("does not allow accessing hidden fields", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("surname")
expect(keys).toContain("address")
expect(keys).not.toContain("age")
for (let i = 0; i < 10; i++) {
await config.api.row.save(table._id!, {
name: generator.first(),
surname: generator.last(),
age: generator.age(),
address: generator.address(),
})
}
})
})
it("does not allow accessing hidden fields even if requested", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).not.toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
afterAll(async () => {
config.end()
})
})
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
},
{ equal: { id: 7 } },
],
},
},
fields: queryFields,
query: {},
fields: ["name", "age"],
})
expect(rows).toHaveLength(expectedRows)
expect(rows).toHaveLength(10)
for (const row of rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
})
}
)
})
})
!isInternal &&
it("will decode _id in oneOf query", async () => {
await config.doInContext(config.appId, async () => {
const result = await search({
tableId: table._id!,
query: {
oneOf: {
_id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
},
},
})
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual(
expect.arrayContaining([1, 4, 8])
)
})
})
it("does not allow accessing hidden fields", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("surname")
expect(keys).toContain("address")
expect(keys).not.toContain("age")
}
})
})
it("does not allow accessing hidden fields even if requested", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).not.toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
})
})
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
},
{ equal: { id: 7 } },
],
},
},
fields: queryFields,
})
expect(rows).toHaveLength(expectedRows)
})
}
)
}
)

View File

@ -0,0 +1,9 @@
const { isDatasourceTest } = require(".")
module.exports = paths => {
return {
filtered: paths
.filter(path => isDatasourceTest(path))
.map(path => ({ test: path })),
}
}

View File

@ -0,0 +1,10 @@
const fs = require("fs")
function isDatasourceTest(path) {
const content = fs.readFileSync(path, "utf8")
return content.includes("datasourceDescribe(")
}
module.exports = {
isDatasourceTest,
}

View File

@ -0,0 +1,9 @@
const { isDatasourceTest } = require(".")
module.exports = paths => {
return {
filtered: paths
.filter(path => !isDatasourceTest(path))
.map(path => ({ test: path })),
}
}