MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into chore/api-typing-2

This commit is contained in:
mike12345567 2024-12-03 12:39:37 +00:00
parent c073500d50 07b5dc6e57
commit 009a2749c5
13 changed files with 766 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.18",
"version": "3.2.19",
"npmClient": "yarn",
"concurrency": 20,
"command": {

4
packages/server/src/api/controllers/auth.ts
View File

@ -2,7 +2,7 @@ import { outputProcessing } from "../../utilities/rowProcessor"
import { InternalTables } from "../../db/utils"
import { getFullUser } from "../../utilities/users"
import { roles, context, db as dbCore } from "@budibase/backend-core"
import { AppSelfResponse, ContextUser, Row, UserCtx } from "@budibase/types"
import { AppSelfResponse, ContextUser, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
import { processUser } from "../../utilities/global"
@ -45,7 +45,7 @@ export async function fetchSelf(ctx: UserCtx<void, AppSelfResponse>) {
try {
const userTable = await sdk.tables.getTable(InternalTables.USER_METADATA)
// specifically needs to make sure is enriched
ctx.body = (await outputProcessing(userTable, user as Row)) as ContextUser
ctx.body = await outputProcessing(userTable, user)
} catch (err: any) {
let response: ContextUser | {}
// user didn't exist in app, don't pretend they do

3
packages/server/src/api/controllers/deploy/index.ts
View File

@ -126,6 +126,9 @@ export async function deploymentProgress(
try {
const db = context.getAppDB()
const deploymentDoc = await db.get<DeploymentDoc>(DocumentType.DEPLOYMENTS)
if (!deploymentDoc.history?.[ctx.params.deploymentId]) {
ctx.throw(404, "No deployment found")
}
ctx.body = deploymentDoc.history?.[ctx.params.deploymentId]
} catch (err) {
ctx.throw(

99
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -169,7 +169,9 @@ const descriptions = datasourceDescribe({
})
if (descriptions.length) {
describe.each(descriptions)("$dbName", ({ config, dsProvider }) => {
describe.each(descriptions)(
"$dbName",
({ config, dsProvider, isOracle, isMSSQL }) => {
let datasource: Datasource
let rawDatasource: Datasource
let client: Knex
@ -209,7 +211,9 @@ if (descriptions.length) {
describe("list", () => {
it("returns all the datasources", async () => {
const datasources = await config.api.datasource.fetch()
expect(datasources).toContainEqual(expect.objectContaining(datasource))
expect(datasources).toContainEqual(
expect.objectContaining(datasource)
)
})
})
@ -310,7 +314,7 @@ if (descriptions.length) {
presence: {
allowEmpty: false,
},
inclusion: [],
inclusion: ["1", "2", "3"],
},
},
[FieldType.NUMBER]: {
@ -412,6 +416,92 @@ if (descriptions.length) {
}
expect(updated).toEqual(expected)
})
!isOracle &&
!isMSSQL &&
it("can fetch options columns with a large number of options", async () => {
const enumOptions = new Array(1000)
.fill(0)
.map((_, i) => i.toString())
.toSorted()
await client.schema.createTable("options", table => {
table.increments("id").primary()
table.enum("enum", enumOptions, {
useNative: true,
enumName: "enum",
})
})
const resp = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(resp.errors).toEqual({})
const table = resp.datasource.entities!.options
expect(
table.schema.enum.constraints!.inclusion!.toSorted()
).toEqual(enumOptions)
})
!isOracle &&
!isMSSQL &&
it("can fetch options with commas in them", async () => {
const enumOptions = [
"Lincoln, Abraham",
"Washington, George",
"Fred",
"Bob",
].toSorted()
await client.schema.createTable("options", table => {
table.increments("id").primary()
table.enum("enum", enumOptions, {
useNative: true,
enumName: "enum",
})
})
const resp = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(resp.errors).toEqual({})
const table = resp.datasource.entities!.options
expect(
table.schema.enum.constraints!.inclusion!.toSorted()
).toEqual(enumOptions)
})
!isOracle &&
!isMSSQL &&
it("can fetch options that may include other type names", async () => {
const enumOptions = [
"int",
"bigint",
"float",
"numeric",
"json",
"map",
].toSorted()
await client.schema.createTable("options", table => {
table.increments("id").primary()
table.enum("enum", enumOptions, {
useNative: true,
enumName: "enum",
})
})
const resp = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(resp.errors).toEqual({})
const table = resp.datasource.entities!.options
expect(
table.schema.enum.constraints!.inclusion!.toSorted()
).toEqual(enumOptions)
})
})
describe("verify", () => {
@ -495,5 +585,6 @@ if (descriptions.length) {
)
})
})
})
}
)
}

4
packages/server/src/automations/actions.ts
View File

@ -96,6 +96,10 @@ if (env.SELF_HOSTED) {
ACTION_IMPLS["EXECUTE_BASH"] = bash.run
// @ts-ignore
BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
if (env.isTest()) {
BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
}
}
export async function getActionDefinitions(): Promise<

152
packages/server/src/automations/tests/bash.spec.ts
View File

@ -1,26 +1,148 @@
import { getConfig, afterAll as _afterAll, runStep } from "./utilities"
import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
import * as automation from "../index"
import * as setup from "./utilities"
import { Table } from "@budibase/types"
describe("test the bash action", () => {
let config = getConfig()
describe("Execute Bash Automations", () => {
let config = setup.getConfig(),
table: Table
beforeAll(async () => {
await automation.init()
await config.init()
table = await config.createTable()
await config.createRow({
name: "test row",
description: "test description",
tableId: table._id!,
})
afterAll(_afterAll)
it("should be able to execute a script", async () => {
let res = await runStep(config, "EXECUTE_BASH", {
code: "echo 'test'",
})
expect(res.stdout).toEqual("test\n")
expect(res.success).toEqual(true)
})
it("should handle a null value", async () => {
let res = await runStep(config, "EXECUTE_BASH", {
code: null,
afterAll(setup.afterAll)
it("should use trigger data in bash command and pass output to subsequent steps", async () => {
const result = await createAutomationBuilder({
name: "Bash with Trigger Data",
config,
})
expect(res.stdout).toEqual(
.appAction({ fields: { command: "hello world" } })
.bash(
{ code: "echo '{{ trigger.fields.command }}'" },
{ stepName: "Echo Command" }
)
.serverLog(
{ text: "Bash output was: {{ steps.[Echo Command].stdout }}" },
{ stepName: "Log Output" }
)
.run()
expect(result.steps[0].outputs.stdout).toEqual("hello world\n")
expect(result.steps[1].outputs.message).toContain(
"Bash output was: hello world"
)
})
it("should chain multiple bash commands using previous outputs", async () => {
const result = await createAutomationBuilder({
name: "Chained Bash Commands",
config,
})
.appAction({ fields: { filename: "testfile.txt" } })
.bash(
{ code: "echo 'initial content' > {{ trigger.fields.filename }}" },
{ stepName: "Create File" }
)
.bash(
{ code: "cat {{ trigger.fields.filename }} | tr '[a-z]' '[A-Z]'" },
{ stepName: "Transform Content" }
)
.bash(
{ code: "rm {{ trigger.fields.filename }}" },
{ stepName: "Cleanup" }
)
.run()
expect(result.steps[1].outputs.stdout).toEqual("INITIAL CONTENT\n")
expect(result.steps[1].outputs.success).toEqual(true)
})
it("should integrate bash output with row operations", async () => {
const result = await createAutomationBuilder({
name: "Bash with Row Operations",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
filters: {},
},
{ stepName: "Get Row" }
)
.bash(
{
code: "echo Row data: {{ steps.[Get Row].rows.[0].name }} - {{ steps.[Get Row].rows.[0].description }}",
},
{ stepName: "Process Row Data" }
)
.serverLog(
{ text: "{{ steps.[Process Row Data].stdout }}" },
{ stepName: "Log Result" }
)
.run()
expect(result.steps[1].outputs.stdout).toContain(
"Row data: test row - test description"
)
expect(result.steps[2].outputs.message).toContain(
"Row data: test row - test description"
)
})
it("should handle bash output in conditional logic", async () => {
const result = await createAutomationBuilder({
name: "Bash with Conditional",
config,
})
.appAction({ fields: { threshold: "5" } })
.bash(
{ code: "echo $(( {{ trigger.fields.threshold }} + 5 ))" },
{ stepName: "Calculate Value" }
)
.executeScript(
{
code: `
const value = parseInt(steps["Calculate Value"].stdout);
return value > 8 ? "high" : "low";
`,
},
{ stepName: "Check Value" }
)
.serverLog(
{ text: "Value was {{ steps.[Check Value].value }}" },
{ stepName: "Log Result" }
)
.run()
expect(result.steps[0].outputs.stdout).toEqual("10\n")
expect(result.steps[1].outputs.value).toEqual("high")
expect(result.steps[2].outputs.message).toContain("Value was high")
})
it("should handle null values gracefully", async () => {
const result = await createAutomationBuilder({
name: "Null Bash Input",
config,
})
.appAction({ fields: {} })
.bash(
//@ts-ignore
{ code: null },
{ stepName: "Null Command" }
)
.run()
expect(result.steps[0].outputs.stdout).toBe(
"Budibase bash automation failed: Invalid inputs"
)
})

69
packages/server/src/automations/tests/openai.spec.ts
View File

@ -1,7 +1,9 @@
import { getConfig, runStep, afterAll as _afterAll } from "./utilities"
import { getConfig, afterAll as _afterAll } from "./utilities"
import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
import { OpenAI } from "openai"
import { setEnv as setCoreEnv } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
import { Model } from "@budibase/types"
jest.mock("openai", () => ({
OpenAI: jest.fn().mockImplementation(() => ({
@ -47,6 +49,7 @@ describe("test the openai action", () => {
let resetEnv: () => void | undefined
beforeAll(async () => {
setCoreEnv({ SELF_HOSTED: true })
await config.init()
})
@ -62,17 +65,39 @@ describe("test the openai action", () => {
afterAll(_afterAll)
it("should be able to receive a response from ChatGPT given a prompt", async () => {
const res = await runStep(config, "OPENAI", { prompt: OPENAI_PROMPT })
expect(res.response).toEqual("This is a test")
expect(res.success).toBeTruthy()
setCoreEnv({ SELF_HOSTED: true })
const result = await createAutomationBuilder({
name: "Test OpenAI Response",
config,
})
.appAction({ fields: {} })
.openai(
{ prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI },
{ stepName: "Basic OpenAI Query" }
)
.run()
expect(result.steps[0].outputs.response).toEqual("This is a test")
expect(result.steps[0].outputs.success).toBeTruthy()
})
it("should present the correct error message when a prompt is not provided", async () => {
const res = await runStep(config, "OPENAI", { prompt: null })
expect(res.response).toEqual(
const result = await createAutomationBuilder({
name: "Test OpenAI No Prompt",
config,
})
.appAction({ fields: {} })
.openai(
{ prompt: "", model: Model.GPT_4O_MINI },
{ stepName: "Empty Prompt Query" }
)
.run()
expect(result.steps[0].outputs.response).toEqual(
"Budibase OpenAI Automation Failed: No prompt supplied"
)
expect(res.success).toBeFalsy()
expect(result.steps[0].outputs.success).toBeFalsy()
})
it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => {
@ -91,14 +116,21 @@ describe("test the openai action", () => {
} as any)
)
const res = await runStep(config, "OPENAI", {
prompt: OPENAI_PROMPT,
const result = await createAutomationBuilder({
name: "Test OpenAI Error",
config,
})
.appAction({ fields: {} })
.openai(
{ prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI },
{ stepName: "Error Producing Query" }
)
.run()
expect(res.response).toEqual(
expect(result.steps[0].outputs.response).toEqual(
"Error: An error occurred while calling createChatCompletion"
)
expect(res.success).toBeFalsy()
expect(result.steps[0].outputs.success).toBeFalsy()
})
it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => {
@ -106,10 +138,19 @@ describe("test the openai action", () => {
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?"
await runStep(config, "OPENAI", {
model: "gpt-4o-mini",
prompt,
await createAutomationBuilder({
name: "Test OpenAI Pro Features",
config,
})
.appAction({ fields: {} })
.openai(
{
model: Model.GPT_4O_MINI,
prompt,
},
{ stepName: "Pro Features Query" }
)
.run()
expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith(
"gpt-4o-mini"

150
packages/server/src/automations/tests/queryRows.spec.ts
View File

@ -1,5 +1,7 @@
import { Table } from "@budibase/types"
import { EmptyFilterOption, SortOrder, Table } from "@budibase/types"
import * as setup from "./utilities"
import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
import * as automation from "../index"
const NAME = "Test"
@ -8,6 +10,7 @@ describe("Test a query step automation", () => {
let config = setup.getConfig()
beforeAll(async () => {
await automation.init()
await config.init()
table = await config.createTable()
const row = {
@ -22,71 +25,92 @@ describe("Test a query step automation", () => {
afterAll(setup.afterAll)
it("should be able to run the query step", async () => {
const inputs = {
tableId: table._id,
const result = await createAutomationBuilder({
name: "Basic Query Test",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
filters: {
equal: {
name: NAME,
},
},
sortColumn: "name",
sortOrder: "ascending",
sortOrder: SortOrder.ASCENDING,
limit: 10,
}
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
},
{ stepName: "Query All Rows" }
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)
expect(res.rows[0].name).toBe(NAME)
.run()
expect(result.steps[0].outputs.success).toBe(true)
expect(result.steps[0].outputs.rows).toBeDefined()
expect(result.steps[0].outputs.rows.length).toBe(2)
expect(result.steps[0].outputs.rows[0].name).toBe(NAME)
})
it("Returns all rows when onEmptyFilter has no value and no filters are passed", async () => {
const inputs = {
tableId: table._id,
const result = await createAutomationBuilder({
name: "Empty Filter Test",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
filters: {},
sortColumn: "name",
sortOrder: "ascending",
sortOrder: SortOrder.ASCENDING,
limit: 10,
}
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
},
{ stepName: "Query With Empty Filter" }
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)
expect(res.rows[0].name).toBe(NAME)
.run()
expect(result.steps[0].outputs.success).toBe(true)
expect(result.steps[0].outputs.rows).toBeDefined()
expect(result.steps[0].outputs.rows.length).toBe(2)
expect(result.steps[0].outputs.rows[0].name).toBe(NAME)
})
it("Returns no rows when onEmptyFilter is RETURN_NONE and theres no filters", async () => {
const inputs = {
tableId: table._id,
const result = await createAutomationBuilder({
name: "Return None Test",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
filters: {},
"filters-def": [],
sortColumn: "name",
sortOrder: "ascending",
sortOrder: SortOrder.ASCENDING,
limit: 10,
onEmptyFilter: "none",
}
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
},
{ stepName: "Query With Return None" }
)
expect(res.success).toBe(false)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0)
.run()
expect(result.steps[0].outputs.success).toBe(false)
expect(result.steps[0].outputs.rows).toBeDefined()
expect(result.steps[0].outputs.rows.length).toBe(0)
})
it("Returns no rows when onEmptyFilters RETURN_NONE and a filter is passed with a null value", async () => {
const inputs = {
tableId: table._id,
onEmptyFilter: "none",
const result = await createAutomationBuilder({
name: "Null Filter Test",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
onEmptyFilter: EmptyFilterOption.RETURN_NONE,
filters: {},
"filters-def": [
{
@ -94,35 +118,39 @@ describe("Test a query step automation", () => {
},
],
sortColumn: "name",
sortOrder: "ascending",
sortOrder: SortOrder.ASCENDING,
limit: 10,
}
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
},
{ stepName: "Query With Null Filter" }
)
expect(res.success).toBe(false)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0)
.run()
expect(result.steps[0].outputs.success).toBe(false)
expect(result.steps[0].outputs.rows).toBeDefined()
expect(result.steps[0].outputs.rows.length).toBe(0)
})
it("Returns rows when onEmptyFilter is RETURN_ALL and no filter is passed", async () => {
const inputs = {
tableId: table._id,
onEmptyFilter: "all",
const result = await createAutomationBuilder({
name: "Return All Test",
config,
})
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
filters: {},
sortColumn: "name",
sortOrder: "ascending",
sortOrder: SortOrder.ASCENDING,
limit: 10,
}
const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
},
{ stepName: "Query With Return All" }
)
expect(res.success).toBe(true)
expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2)
.run()
expect(result.steps[0].outputs.success).toBe(true)
expect(result.steps[0].outputs.rows).toBeDefined()
expect(result.steps[0].outputs.rows.length).toBe(2)
})
})

26
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View File

@ -35,6 +35,8 @@ import {
Branch,
FilterStepInputs,
ExecuteScriptStepInputs,
OpenAIStepInputs,
BashStepInputs,
} from "@budibase/types"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import * as setup from "../utilities"
@ -221,6 +223,30 @@ class BaseStepBuilder {
input
)
}
bash(
input: BashStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.EXECUTE_BASH,
BUILTIN_ACTION_DEFINITIONS.EXECUTE_BASH,
input,
opts
)
}
openai(
input: OpenAIStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.OPENAI,
BUILTIN_ACTION_DEFINITIONS.OPENAI,
input,
opts
)
}
}
class StepBuilder extends BaseStepBuilder {
build(): AutomationStep[] {

4
packages/server/src/integrations/mysql.ts
View File

@ -322,9 +322,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
presence: required && !isAuto && !hasDefault,
externalType: column.Type,
options: column.Type.startsWith("enum")
? column.Type.substring(5, column.Type.length - 1)
.split(",")
.map(str => str.replace(/^'(.*)'$/, "$1"))
? column.Type.substring(6, column.Type.length - 2).split("','")
: undefined,
})
}

12
packages/server/src/integrations/utils/utils.ts
View File

@ -138,12 +138,22 @@ export function generateColumnDefinition(config: {
let { externalType, autocolumn, name, presence, options } = config
let foundType = FieldType.STRING
const lowerCaseType = externalType.toLowerCase()
let matchingTypes = []
let matchingTypes: { external: string; internal: PrimitiveTypes }[] = []
// In at least MySQL, the external type of an ENUM column is "enum('option1',
// 'option2', ...)", which can potentially contain any type name as a
// substring. To get around this interfering with the loop below, we first
// check for an enum column and handle that separately.
if (lowerCaseType.startsWith("enum")) {
matchingTypes.push({ external: "enum", internal: FieldType.OPTIONS })
} else {
for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
if (lowerCaseType.includes(external)) {
matchingTypes.push({ external, internal })
}
}
}
// Set the foundType based the longest match
if (matchingTypes.length > 0) {
foundType = matchingTypes.reduce((acc, val) => {

4
packages/types/src/api/web/deployment.ts
View File

@ -2,13 +2,11 @@ import { DeploymentDoc, DeploymentStatus } from "../../documents"
export interface PublishAppResponse extends DeploymentDoc {}
export type DeploymentProgressResponse =
| {
export interface DeploymentProgressResponse {
_id: string
appId: string
status?: DeploymentStatus
updatedAt: number
}
| undefined
export type FetchDeploymentResponse = DeploymentProgressResponse[]

2
packages/types/src/documents/app/automation/StepInputsOutputs.ts
View File

@ -150,7 +150,7 @@ export type OpenAIStepInputs = {
prompt: string
model: Model
}
enum Model {
export enum Model {
GPT_35_TURBO = "gpt-3.5-turbo",
// will only work with api keys that have access to the GPT4 API
GPT_4 = "gpt-4",