From 84f52683b23f4f3d3078094b9de43f7e239b85a6 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:55:44 +0100 Subject: [PATCH 01/24] chatgpt automation block --- packages/server/package.json | 1 + packages/server/src/automations/actions.ts | 3 + .../server/src/automations/steps/openai.ts | 104 ++++++++++++++++++ .../src/automations/tests/openai.spec.ts | 85 ++++++++++++++ packages/server/src/environment.ts | 1 + .../types/src/documents/app/automation.ts | 1 + yarn.lock | 20 +++- 7 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/automations/steps/openai.ts create mode 100644 packages/server/src/automations/tests/openai.spec.ts diff --git a/packages/server/package.json b/packages/server/package.json index 6aadfd15a0..2e53c0e7ac 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -99,6 +99,7 @@ "mysql2": "2.3.3", "node-fetch": "2.6.7", "open": "8.4.0", + "openai": "^3.2.1", "pg": "8.5.1", "posthog-node": "1.3.0", "pouchdb": "7.3.0", diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 2a6b760725..de92efb676 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -14,6 +14,7 @@ import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" +import * as openai from "./steps/openai" import env from "../environment" import { AutomationStepSchema, @@ -39,6 +40,7 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, + OPEN_AI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -59,6 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, + OPEN_AI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts new file mode 100644 index 0000000000..79586bb712 --- /dev/null +++ b/packages/server/src/automations/steps/openai.ts @@ -0,0 +1,104 @@ +import { Configuration, OpenAIApi } from "openai"; +import { + AutomationActionStepId, + AutomationStepSchema, + AutomationStepInput, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" +import * as automationUtils from "../automationUtils" +import environment from "../../environment"; + +enum Model { + GPT_35_TURBO = "gpt-3.5-turbo", + // will only work with api keys that have access to the GPT4 API + // GPT_4 = "gpt-4", +} + +export const definition: AutomationStepSchema = { + name: "OpenAI", + tagline: "Send prompts to ChatGPT", + icon: "Algorithm", + description: "Interact with the OpenAI ChatGPT API.", + type: AutomationStepType.ACTION, + internal: true, + stepId: AutomationActionStepId.OPEN_AI, + inputs: { + prompt: "", + }, + schema: { + inputs: { + properties: { + prompt: { + type: AutomationIOType.STRING, + title: "Prompt", + }, + model: { + type: AutomationIOType.STRING, + title: "Model", + enum: Object.values(Model), + }, + }, + required: ["prompt", "model"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + response: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "response"], + }, + }, +} + +export async function run({ inputs, context }: AutomationStepInput) { + if (!environment.OPENAI_API_KEY) { + return { + success: false, + response: "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + + if (inputs.prompt == null) { + return { + success: false, + response: "Budibase OpenAI Automation Failed: No prompt supplied", + } + } + + try { + const configuration = new Configuration({ + apiKey: environment.OPENAI_API_KEY, + }); + + const openai = new OpenAIApi(configuration); + + const completion = await openai.createChatCompletion({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt + } + ], + }); + + let response = completion?.data?.choices[0]?.message?.content + + return { + response, + success: true, + } + } catch (err) { + return { + success: false, + response: automationUtils.getError(err), + } + } +} diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts new file mode 100644 index 0000000000..3ba9463f21 --- /dev/null +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -0,0 +1,85 @@ +const setup = require("./utilities") +import environment from "../../environment"; +import openai from "openai" + +jest.mock("openai", jest.fn(() => ({ + Configuration: jest.fn(), + OpenAIApi: jest.fn(() => ({ + createChatCompletion: jest.fn(() => ({ + data: { + choices: [ + { + message: { + content: "This is a test" + }, + } + ] + } + })) + })) +}))) + +const OPENAI_PROMPT = "What is the meaning of life?" + +describe("test the openai action", () => { + let config = setup.getConfig() + + beforeAll(async () => { + await config.init() + }) + + beforeEach(() => { + environment.OPENAI_API_KEY = "abc123" + }) + + afterAll(setup.afterAll) + + + it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { + delete environment.OPENAI_API_KEY + + let res = await setup.runStep("OPEN_AI", + { + prompt: OPENAI_PROMPT + } + ) + expect(res.response).toEqual("OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.") + expect(res.success).toBeFalsy() + }) + + it("should be able to receive a response from ChatGPT given a prompt", async () => { + const res = await setup.runStep("OPEN_AI", + { + prompt: OPENAI_PROMPT + } + ) + expect(res.response).toEqual("This is a test") + expect(res.success).toBeTruthy() + }) + + + it("should present the correct error message when a prompt is not provided", async () => { + const res = await setup.runStep("OPEN_AI", + { + prompt: null + } + ) + expect(res.response).toEqual("Budibase OpenAI Automation Failed: No prompt supplied") + expect(res.success).toBeFalsy() + }) + + it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { + openai.OpenAIApi.mockImplementation(() => ({ + createChatCompletion: jest.fn(() => { + throw new Error("An error occurred while calling createChatCompletion"); + }), + })); + + const res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }); + + expect(res.response).toEqual("Error: An error occurred while calling createChatCompletion") + expect(res.success).toBeFalsy() + }); +}) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 1bd5a6486c..9a52b18e08 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -72,6 +72,7 @@ const environment = { BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", + OPENAI_API_KEY: process.env.OPENAI_API_KEY, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index aa600c6375..eaff533761 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -56,6 +56,7 @@ export enum AutomationActionStepId { FILTER = "FILTER", QUERY_ROWS = "QUERY_ROWS", LOOP = "LOOP", + OPEN_AI = "OPEN_AI", // these used to be lowercase step IDs, maintain for backwards compat discord = "discord", slack = "slack", diff --git a/yarn.lock b/yarn.lock index 260f0ae6a6..3d233dbe76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.5.6-alpha.29": - version "2.5.6-alpha.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.29.tgz#71414f68a296535ef53ffb0453352ea137c4aeab" - integrity sha512-tQuzMOo2WFxKvsUgYAfUEcLabRpmAD7hPlhBhCFzYasaXNbJiPhcwv4i52US0i0Wr2IXMb2X0d7fwa8tnbKzIA== +"@budibase/pro@2.5.6-alpha.30": + version "2.5.6-alpha.30" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.30.tgz#9b8089a983fd61a062f31a8e5757d7bb5b56fb8c" + integrity sha512-YTyjMHK/wsSOFJkON7a5WRJSgAr8Gh/cflRzifm6Jw1Gb8S8B8Z6uTWW/S7+psVBRGeUfV1s8biYNr71tXz2Ng== dependencies: - "@budibase/backend-core" "2.5.6-alpha.29" + "@budibase/backend-core" "2.5.6-alpha.30" "@budibase/shared-core" "2.4.44-alpha.1" "@budibase/string-templates" "2.4.44-alpha.1" - "@budibase/types" "2.5.6-alpha.29" + "@budibase/types" "2.5.6-alpha.30" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -18358,6 +18358,14 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" +openai@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/openai/-/openai-3.2.1.tgz#1fa35bdf979cbde8453b43f2dd3a7d401ee40866" + integrity sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A== + dependencies: + axios "^0.26.0" + form-data "^4.0.0" + openapi-response-validator@^9.2.0: version "9.3.1" resolved "https://registry.yarnpkg.com/openapi-response-validator/-/openapi-response-validator-9.3.1.tgz#54284d8be608ef53283cbe7448accce8106b1c56" From 4f020a4db4766974d092827e8c843c2a79050fb1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:56:46 +0100 Subject: [PATCH 02/24] lint --- .../server/src/automations/steps/openai.ts | 17 +++-- .../src/automations/tests/openai.spec.ts | 75 ++++++++++--------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 79586bb712..a7c22ffd0c 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -1,4 +1,4 @@ -import { Configuration, OpenAIApi } from "openai"; +import { Configuration, OpenAIApi } from "openai" import { AutomationActionStepId, AutomationStepSchema, @@ -7,7 +7,7 @@ import { AutomationIOType, } from "@budibase/types" import * as automationUtils from "../automationUtils" -import environment from "../../environment"; +import environment from "../../environment" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", @@ -61,7 +61,8 @@ export async function run({ inputs, context }: AutomationStepInput) { if (!environment.OPENAI_API_KEY) { return { success: false, - response: "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", } } @@ -75,19 +76,19 @@ export async function run({ inputs, context }: AutomationStepInput) { try { const configuration = new Configuration({ apiKey: environment.OPENAI_API_KEY, - }); + }) - const openai = new OpenAIApi(configuration); + const openai = new OpenAIApi(configuration) const completion = await openai.createChatCompletion({ model: inputs.model, messages: [ { role: "user", - content: inputs.prompt - } + content: inputs.prompt, + }, ], - }); + }) let response = completion?.data?.choices[0]?.message?.content diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3ba9463f21..032f670db1 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -1,23 +1,26 @@ const setup = require("./utilities") -import environment from "../../environment"; +import environment from "../../environment" import openai from "openai" -jest.mock("openai", jest.fn(() => ({ - Configuration: jest.fn(), - OpenAIApi: jest.fn(() => ({ - createChatCompletion: jest.fn(() => ({ - data: { - choices: [ - { - message: { - content: "This is a test" +jest.mock( + "openai", + jest.fn(() => ({ + Configuration: jest.fn(), + OpenAIApi: jest.fn(() => ({ + createChatCompletion: jest.fn(() => ({ + data: { + choices: [ + { + message: { + content: "This is a test", + }, }, - } - ] - } - })) + ], + }, + })), + })), })) -}))) +) const OPENAI_PROMPT = "What is the meaning of life?" @@ -34,52 +37,50 @@ describe("test the openai action", () => { afterAll(setup.afterAll) - it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { delete environment.OPENAI_API_KEY - let res = await setup.runStep("OPEN_AI", - { - prompt: OPENAI_PROMPT - } + let res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }) + expect(res.response).toEqual( + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable." ) - expect(res.response).toEqual("OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.") expect(res.success).toBeFalsy() }) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await setup.runStep("OPEN_AI", - { - prompt: OPENAI_PROMPT - } - ) + const res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }) expect(res.response).toEqual("This is a test") expect(res.success).toBeTruthy() }) - it("should present the correct error message when a prompt is not provided", async () => { - const res = await setup.runStep("OPEN_AI", - { - prompt: null - } + const res = await setup.runStep("OPEN_AI", { + prompt: null, + }) + expect(res.response).toEqual( + "Budibase OpenAI Automation Failed: No prompt supplied" ) - expect(res.response).toEqual("Budibase OpenAI Automation Failed: No prompt supplied") expect(res.success).toBeFalsy() }) it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { openai.OpenAIApi.mockImplementation(() => ({ createChatCompletion: jest.fn(() => { - throw new Error("An error occurred while calling createChatCompletion"); + throw new Error("An error occurred while calling createChatCompletion") }), - })); + })) const res = await setup.runStep("OPEN_AI", { prompt: OPENAI_PROMPT, - }); + }) - expect(res.response).toEqual("Error: An error occurred while calling createChatCompletion") + expect(res.response).toEqual( + "Error: An error occurred while calling createChatCompletion" + ) expect(res.success).toBeFalsy() - }); + }) }) From e70e3ae662fee5fa11e1123bf787640f9e750627 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:58:21 +0100 Subject: [PATCH 03/24] rename --- packages/server/src/automations/actions.ts | 4 ++-- packages/server/src/automations/steps/openai.ts | 4 ++-- packages/server/src/automations/tests/openai.spec.ts | 8 ++++---- packages/types/src/documents/app/automation.ts | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index de92efb676..f0feab006c 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -40,7 +40,7 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, - OPEN_AI: openai.run, + OPENAI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -61,7 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, - OPEN_AI: openai.definition, + OPENAI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index a7c22ffd0c..265a40d466 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -22,7 +22,7 @@ export const definition: AutomationStepSchema = { description: "Interact with the OpenAI ChatGPT API.", type: AutomationStepType.ACTION, internal: true, - stepId: AutomationActionStepId.OPEN_AI, + stepId: AutomationActionStepId.OPENAI, inputs: { prompt: "", }, @@ -90,7 +90,7 @@ export async function run({ inputs, context }: AutomationStepInput) { ], }) - let response = completion?.data?.choices[0]?.message?.content + const response = completion?.data?.choices[0]?.message?.content return { response, diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 032f670db1..31f7e48305 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -40,7 +40,7 @@ describe("test the openai action", () => { it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { delete environment.OPENAI_API_KEY - let res = await setup.runStep("OPEN_AI", { + let res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) expect(res.response).toEqual( @@ -50,7 +50,7 @@ describe("test the openai action", () => { }) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) expect(res.response).toEqual("This is a test") @@ -58,7 +58,7 @@ describe("test the openai action", () => { }) it("should present the correct error message when a prompt is not provided", async () => { - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: null, }) expect(res.response).toEqual( @@ -74,7 +74,7 @@ describe("test the openai action", () => { }), })) - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index eaff533761..11e868023f 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -56,7 +56,7 @@ export enum AutomationActionStepId { FILTER = "FILTER", QUERY_ROWS = "QUERY_ROWS", LOOP = "LOOP", - OPEN_AI = "OPEN_AI", + OPENAI = "OPENAI", // these used to be lowercase step IDs, maintain for backwards compat discord = "discord", slack = "slack", From b112995fd05cb86a7bbe9dc37bd2e248d9c12a17 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 15:26:20 +0100 Subject: [PATCH 04/24] Disable hide column option in header cell context menu for sticky column --- .../src/components/grid/cells/HeaderCell.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 72b0ad0ff1..21ee210233 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -196,7 +196,11 @@ Move right - Hide column + Hide column From 0a0e78c314d4cda86cfd9168341227f3f225f77d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 15:29:48 +0100 Subject: [PATCH 05/24] Fix grid keyboard shortcuts being hard to read in light theme --- .../src/components/grid/layout/KeyboardShortcut.svelte | 2 +- packages/frontend-core/src/components/grid/layout/NewRow.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte b/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte index 5024a24ea7..cac39bbf2f 100644 --- a/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte +++ b/packages/frontend-core/src/components/grid/layout/KeyboardShortcut.svelte @@ -38,7 +38,7 @@ padding: 2px 6px; font-size: 12px; font-weight: 600; - background-color: var(--spectrum-global-color-gray-200); + background-color: var(--spectrum-global-color-gray-300); color: var(--spectrum-global-color-gray-700); border-radius: 4px; text-align: center; diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index 8048a4e2fa..f705881a8d 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -219,7 +219,7 @@ From 834202423f920d0ab1b4ad982b6e1f3a93a785f6 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 16:42:36 +0100 Subject: [PATCH 06/24] Fix bulk deletion triggering on delete keypress after selecting then deselecting a row --- .../components/grid/cells/GutterCell.svelte | 11 +--------- .../grid/overlays/KeyboardManager.svelte | 5 +---- .../src/components/grid/stores/ui.js | 21 ++++++++++++++++++- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte index 00b99c0711..588818f10d 100644 --- a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte @@ -21,16 +21,7 @@ svelteDispatch("select") const id = row?._id if (id) { - selectedRows.update(state => { - let newState = { - ...state, - [id]: !state[id], - } - if (!newState[id]) { - delete newState[id] - } - return newState - }) + selectedRows.actions.toggleRow(id) } } diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte index c7fa0a5cb7..6d16acc7c5 100644 --- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte +++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte @@ -224,10 +224,7 @@ if (!id || id === NewRowID) { return } - selectedRows.update(state => { - state[id] = !state[id] - return state - }) + selectedRows.actions.toggleRow(id) } onMount(() => { diff --git a/packages/frontend-core/src/components/grid/stores/ui.js b/packages/frontend-core/src/components/grid/stores/ui.js index 85afad8a90..b62e883437 100644 --- a/packages/frontend-core/src/components/grid/stores/ui.js +++ b/packages/frontend-core/src/components/grid/stores/ui.js @@ -25,14 +25,33 @@ export const createStores = () => { null ) + // Toggles whether a certain row ID is selected or not + const toggleSelectedRow = id => { + selectedRows.update(state => { + let newState = { + ...state, + [id]: !state[id], + } + if (!newState[id]) { + delete newState[id] + } + return newState + }) + } + return { focusedCellId, focusedCellAPI, focusedRowId, previousFocusedRowId, - selectedRows, hoveredRowId, rowHeight, + selectedRows: { + ...selectedRows, + actions: { + toggleRow: toggleSelectedRow, + }, + }, } } From ba9691ee12aa04d31089db3dc90690dfb079a0c0 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 17:09:32 +0100 Subject: [PATCH 07/24] Add grid flag to stripe rows different colours --- .../src/components/grid/cells/DataCell.svelte | 2 ++ .../src/components/grid/cells/GridCell.svelte | 13 +++++++++---- .../src/components/grid/cells/GutterCell.svelte | 1 + .../src/components/grid/layout/Grid.svelte | 8 ++++++++ .../src/components/grid/layout/GridBody.svelte | 6 +++++- .../src/components/grid/layout/GridRow.svelte | 5 +++-- .../src/components/grid/layout/NewRow.svelte | 4 ++-- .../src/components/grid/layout/StickyColumn.svelte | 3 ++- 8 files changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte index f39b820632..cb8616a735 100644 --- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte @@ -11,6 +11,7 @@ export let selected export let rowFocused export let rowIdx + export let topRow = false export let focused export let selectedUser export let column @@ -68,6 +69,7 @@ {highlighted} {selected} {rowIdx} + {topRow} {focused} {selectedUser} {readonly} diff --git a/packages/frontend-core/src/components/grid/cells/GridCell.svelte b/packages/frontend-core/src/components/grid/cells/GridCell.svelte index 6589c18d07..7e38a989d6 100644 --- a/packages/frontend-core/src/components/grid/cells/GridCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GridCell.svelte @@ -6,6 +6,7 @@ export let selectedUser = null export let error = null export let rowIdx + export let topRow = false export let defaultHeight = false export let center = false export let readonly = false @@ -31,13 +32,14 @@ class:readonly class:default-height={defaultHeight} class:selected-other={selectedUser != null} + class:alt={rowIdx % 2 === 1} + class:top={topRow} on:focus on:mousedown on:mouseup on:click on:contextmenu {style} - data-row={rowIdx} > {#if error}
@@ -70,6 +72,9 @@ width: 0; --cell-color: transparent; } + .cell.alt { + --cell-background: var(--cell-background-alt); + } .cell.default-height { height: var(--default-row-height); } @@ -98,8 +103,8 @@ .cell.selected-other:not(.focused):after { border-radius: 0 2px 2px 2px; } - .cell[data-row="0"].error:after, - .cell[data-row="0"].selected-other:not(.focused):after { + .cell.top.error:after, + .cell.top.selected-other:not(.focused):after { border-radius: 2px 2px 2px 0; } @@ -152,7 +157,7 @@ overflow: hidden; user-select: none; } - .cell[data-row="0"] .label { + .cell.top .label { bottom: auto; top: 100%; border-radius: 0 2px 2px 2px; diff --git a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte index 588818f10d..56c4c20d54 100644 --- a/packages/frontend-core/src/components/grid/cells/GutterCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/GutterCell.svelte @@ -38,6 +38,7 @@ highlighted={rowFocused || rowHovered} selected={rowSelected} {defaultHeight} + rowIdx={row?.__idx} >
{#if $$slots.default} diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index 2035ec4d39..314479f519 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -40,6 +40,7 @@ export let allowExpandRows = true export let allowEditRows = true export let allowDeleteRows = true + export let stripeRows = false // Unique identifier for DOM nodes inside this instance const rand = Math.random() @@ -54,6 +55,7 @@ allowExpandRows, allowEditRows, allowDeleteRows, + stripeRows, }) // Build up context @@ -88,6 +90,7 @@ allowExpandRows, allowEditRows, allowDeleteRows, + stripeRows, }) // Set context for children to consume @@ -105,6 +108,7 @@ id="grid-{rand}" class:is-resizing={$isResizing} class:is-reordering={$isReordering} + class:stripe={$config.stripeRows} style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-height:{MaxCellRenderHeight}px; --max-cell-render-width-overflow:{MaxCellRenderWidthOverflow}px; --content-lines:{$contentLines};" >
@@ -167,6 +171,7 @@ /* Variables */ --cell-background: var(--spectrum-global-color-gray-50); --cell-background-hover: var(--spectrum-global-color-gray-100); + --cell-background-alt: var(--cell-background); --cell-padding: 8px; --cell-spacing: 4px; --cell-border: 1px solid var(--spectrum-global-color-gray-200); @@ -183,6 +188,9 @@ .grid.is-reordering :global(*) { cursor: grabbing !important; } + .grid.stripe { + --cell-background-alt: var(--spectrum-global-color-gray-75); + } .grid-data-outer, .grid-data-inner { diff --git a/packages/frontend-core/src/components/grid/layout/GridBody.svelte b/packages/frontend-core/src/components/grid/layout/GridBody.svelte index 67f5f03898..016369df49 100644 --- a/packages/frontend-core/src/components/grid/layout/GridBody.svelte +++ b/packages/frontend-core/src/components/grid/layout/GridBody.svelte @@ -36,7 +36,11 @@
{#each $renderedRows as row, idx} - = $rowVerticalInversionIndex} /> + = $rowVerticalInversionIndex} + /> {/each} {#if $config.allowAddRows && $renderedColumns.length}
= $columnHorizontalInversionIndex} highlighted={rowHovered || rowFocused || reorderSource === column.name} selected={rowSelected} - rowIdx={idx} + rowIdx={row.__idx} + topRow={top} focused={$focusedCellId === cellId} selectedUser={$selectedCellMap[cellId]} width={column.width} diff --git a/packages/frontend-core/src/components/grid/layout/NewRow.svelte b/packages/frontend-core/src/components/grid/layout/NewRow.svelte index f705881a8d..85b430f79b 100644 --- a/packages/frontend-core/src/components/grid/layout/NewRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/NewRow.svelte @@ -167,7 +167,7 @@ focused={$focusedCellId === cellId} width={$stickyColumn.width} {updateValue} - rowIdx={0} + topRow={offset === 0} {invertY} > {#if $stickyColumn?.schema?.autocolumn} @@ -193,7 +193,7 @@ row={newRow} focused={$focusedCellId === cellId} width={column.width} - rowIdx={0} + topRow={offset === 0} invertX={columnIdx >= $columnHorizontalInversionIndex} {invertY} > diff --git a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte index 6301112110..801772ed51 100644 --- a/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte +++ b/packages/frontend-core/src/components/grid/layout/StickyColumn.svelte @@ -82,7 +82,8 @@ {rowFocused} selected={rowSelected} highlighted={rowHovered || rowFocused} - rowIdx={idx} + rowIdx={row.__idx} + topRow={idx === 0} focused={$focusedCellId === cellId} selectedUser={$selectedCellMap[cellId]} width={$stickyColumn.width} From cadd1b5a4e2a4491277bf07d3b456ce36e42576b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 16 May 2023 19:48:14 +0100 Subject: [PATCH 08/24] Add automatic scrolling left/right when dragging to reorder columns --- .../src/components/grid/stores/reorder.js | 86 +++++++++++++++++-- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/reorder.js b/packages/frontend-core/src/components/grid/stores/reorder.js index de343987db..a99c1b1ab2 100644 --- a/packages/frontend-core/src/components/grid/stores/reorder.js +++ b/packages/frontend-core/src/components/grid/stores/reorder.js @@ -4,9 +4,10 @@ const reorderInitialState = { sourceColumn: null, targetColumn: null, breakpoints: [], - initialMouseX: null, - scrollLeft: 0, gridLeft: 0, + width: 0, + latestX: 0, + increment: 0, } export const createStores = () => { @@ -23,14 +24,24 @@ export const createStores = () => { } export const deriveStores = context => { - const { reorder, columns, visibleColumns, scroll, bounds, stickyColumn, ui } = - context + const { + reorder, + columns, + visibleColumns, + scroll, + bounds, + stickyColumn, + ui, + maxScrollLeft, + } = context + + let autoScrollInterval + let isAutoScrolling // Callback when dragging on a colum header and starting reordering const startReordering = (column, e) => { const $visibleColumns = get(visibleColumns) const $bounds = get(bounds) - const $scroll = get(scroll) const $stickyColumn = get(stickyColumn) ui.actions.blur() @@ -51,9 +62,8 @@ export const deriveStores = context => { sourceColumn: column, targetColumn: null, breakpoints, - initialMouseX: e.clientX, - scrollLeft: $scroll.left, gridLeft: $bounds.left, + width: $bounds.width, }) // Add listeners to handle mouse movement @@ -66,12 +76,44 @@ export const deriveStores = context => { // Callback when moving the mouse when reordering columns const onReorderMouseMove = e => { + // Immediately handle the current position + const x = e.clientX + reorder.update(state => ({ + ...state, + latestX: x, + })) + considerReorderPosition() + + // Check if we need to start auto-scrolling const $reorder = get(reorder) + const proximityCutoff = 140 + const speedFactor = 8 + const rightProximity = Math.max(0, $reorder.gridLeft + $reorder.width - x) + const leftProximity = Math.max(0, x - $reorder.gridLeft) + if (rightProximity < proximityCutoff) { + const weight = proximityCutoff - rightProximity + const increment = (weight / proximityCutoff) * speedFactor + reorder.update(state => ({ ...state, increment })) + startAutoScroll() + } else if (leftProximity < proximityCutoff) { + const weight = -1 * (proximityCutoff - leftProximity) + const increment = (weight / proximityCutoff) * speedFactor + reorder.update(state => ({ ...state, increment })) + startAutoScroll() + } else { + stopAutoScroll() + } + } + + // Actual logic to consider the current position and determine the new order + const considerReorderPosition = () => { + const $reorder = get(reorder) + const $scroll = get(scroll) // Compute the closest breakpoint to the current position let targetColumn let minDistance = Number.MAX_SAFE_INTEGER - const mouseX = e.clientX - $reorder.gridLeft + $reorder.scrollLeft + const mouseX = $reorder.latestX - $reorder.gridLeft + $scroll.left $reorder.breakpoints.forEach(point => { const distance = Math.abs(point.x - mouseX) if (distance < minDistance) { @@ -79,7 +121,6 @@ export const deriveStores = context => { targetColumn = point.column } }) - if (targetColumn !== $reorder.targetColumn) { reorder.update(state => ({ ...state, @@ -88,8 +129,35 @@ export const deriveStores = context => { } } + // Commences auto-scrolling in a certain direction, triggered when the mouse + // approaches the edges of the grid + const startAutoScroll = () => { + if (isAutoScrolling) { + return + } + isAutoScrolling = true + autoScrollInterval = setInterval(() => { + const $maxLeft = get(maxScrollLeft) + const { increment } = get(reorder) + scroll.update(state => ({ + ...state, + left: Math.max(0, Math.min($maxLeft, state.left + increment)), + })) + considerReorderPosition() + }, 10) + } + + // Stops auto scrolling + const stopAutoScroll = () => { + isAutoScrolling = false + clearInterval(autoScrollInterval) + } + // Callback when stopping reordering columns const stopReordering = async () => { + // Ensure auto-scrolling is stopped + stopAutoScroll() + // Swap position of columns let { sourceColumn, targetColumn } = get(reorder) moveColumn(sourceColumn, targetColumn) From efb45a850b626c485d7e347d2aa3ec867e5ab0a8 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 17 May 2023 08:07:07 +0100 Subject: [PATCH 09/24] Update z-indexes for add column button --- .../frontend-core/src/components/grid/layout/HeaderRow.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte index be9fad00d0..9d6cc2275b 100644 --- a/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte +++ b/packages/frontend-core/src/components/grid/layout/HeaderRow.svelte @@ -61,7 +61,7 @@ border-right: var(--cell-border); border-bottom: var(--cell-border); background: var(--spectrum-global-color-gray-100); - z-index: 20; + z-index: 1; } .add:hover { background: var(--spectrum-global-color-gray-200); From 475c962a5fdaab584626e57b051d136b64c44744 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 17 May 2023 08:16:47 +0100 Subject: [PATCH 10/24] Center align boolean fields --- .../frontend-core/src/components/grid/cells/BooleanCell.svelte | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte index 52aecb07a7..c3449c0b39 100644 --- a/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BooleanCell.svelte @@ -37,6 +37,9 @@ .boolean-cell { padding: 2px var(--cell-padding); pointer-events: none; + flex: 1 1 auto; + display: flex; + justify-content: center; } .boolean-cell.editable { pointer-events: all; From db5d051755bf1017b093a4147a4d62109870b370 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 May 2023 15:26:27 +0100 Subject: [PATCH 11/24] GPT4 support --- packages/server/src/automations/steps/openai.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 265a40d466..88d3fb8b85 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -12,7 +12,7 @@ import environment from "../../environment" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API - // GPT_4 = "gpt-4", + GPT_4 = "gpt-4", } export const definition: AutomationStepSchema = { From f86d321e2ea7978057f4e2231949dd3bc96c4f36 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 May 2023 17:17:23 +0100 Subject: [PATCH 12/24] restrict openai just to self host --- packages/server/src/automations/actions.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 01a9731d20..2f64e75816 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -14,7 +14,6 @@ import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" -import * as openai from "./steps/openai" import env from "../environment" import { AutomationStepSchema, @@ -40,7 +39,6 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, - OPENAI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -61,7 +59,6 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, - OPENAI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, @@ -74,10 +71,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = // ran at all if (env.SELF_HOSTED) { const bash = require("./steps/bash") + const openai = require("./steps/openai") + // @ts-ignore ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + + ACTION_IMPLS.OPENAI = openai.run + BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition } export async function getActionDefinitions() { From 429dc8f6d2b52350e64cfb14401bfbb96d20e23f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 11:49:18 +0200 Subject: [PATCH 13/24] Tidy up tsconfigs --- packages/server/tsconfig.build.json | 2 +- packages/server/tsconfig.json | 8 +------- packages/worker/tsconfig.json | 8 +------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/server/tsconfig.build.json b/packages/server/tsconfig.build.json index 355b7ed6da..bbce6a8bda 100644 --- a/packages/server/tsconfig.build.json +++ b/packages/server/tsconfig.build.json @@ -8,7 +8,7 @@ "esModuleInterop": true, "resolveJsonModule": true, "incremental": true, - "types": ["node", "jest"], + "types": ["node"], "outDir": "dist/src", "skipLibCheck": true, "baseUrl": ".", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index cf86094cd2..849e1e1a83 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -6,13 +6,7 @@ "sourceMap": true, "baseUrl": ".", "outDir": "dist", - "paths": { - "@budibase/types": ["../types/src"], - "@budibase/backend-core": ["../backend-core/src"], - "@budibase/backend-core/*": ["../backend-core/*"], - "@budibase/shared-core": ["../shared-core/src"], - "@budibase/pro": ["../pro/packages/pro/src"] - } + "types": ["node", "jest"] }, "ts-node": { "require": ["tsconfig-paths/register"], diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json index 61f3f9f5e1..147ef1f700 100644 --- a/packages/worker/tsconfig.json +++ b/packages/worker/tsconfig.json @@ -4,13 +4,7 @@ "composite": true, "declaration": true, "sourceMap": true, - "baseUrl": ".", - "paths": { - "@budibase/types": ["../types/src"], - "@budibase/backend-core": ["../backend-core/src"], - "@budibase/backend-core/*": ["../backend-core/*"], - "@budibase/pro": ["../pro/packages/pro/src"] - } + "baseUrl": "." }, "ts-node": { "require": ["tsconfig-paths/register"], From bdda73a70bb91e42babd81ee5eeccfd38d0072b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 11:49:38 +0200 Subject: [PATCH 14/24] Respect tsconfig paths --- package.json | 1 + scripts/build.js | 4 ++++ yarn.lock | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 277b213b78..edbb82b892 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "private": true, "devDependencies": { "@esbuild-plugins/node-resolve": "^0.2.2", + "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@nx/esbuild": "16.2.1", "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", diff --git a/scripts/build.js b/scripts/build.js index 986dbad13f..34f0a70d35 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -9,6 +9,9 @@ const path = require("path") const { build } = require("esbuild") const { default: NodeResolve } = require("@esbuild-plugins/node-resolve") +const { + default: TsconfigPathsPlugin, +} = require("@esbuild-plugins/tsconfig-paths") var argv = require("minimist")(process.argv.slice(2)) @@ -23,6 +26,7 @@ function runBuild(entry, outfile) { sourcemap: isDev, tsconfig, plugins: [ + TsconfigPathsPlugin({ tsconfig }), NodeResolve({ extensions: [".ts", ".js"], onResolved: resolved => { diff --git a/yarn.lock b/yarn.lock index 086342b0f2..e044074a50 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2047,6 +2047,15 @@ escape-string-regexp "^4.0.0" resolve "^1.19.0" +"@esbuild-plugins/tsconfig-paths@^0.1.2": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@esbuild-plugins/tsconfig-paths/-/tsconfig-paths-0.1.2.tgz#1955de0a124ecf4364717a2fadbfbea876955232" + integrity sha512-TusFR26Y+Ze+Zm+NdfqZTSG4XyrXKxIaAfYCL3jASEI/gHjSdoCujATjzNWaaXs6Sk6Bv2D7NLr4Jdz1gysy/Q== + dependencies: + debug "^4.3.1" + find-up "^5.0.0" + strip-json-comments "^3.1.1" + "@esbuild/android-arm64@0.16.17": version "0.16.17" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.16.17.tgz#cf91e86df127aa3d141744edafcba0abdc577d23" @@ -12085,6 +12094,14 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + findit2@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" @@ -17012,6 +17029,13 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + lodash-es@^4.17.11: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" @@ -19371,7 +19395,7 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.1.0: +p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -19406,6 +19430,13 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map-series@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map-series/-/p-map-series-2.1.0.tgz#7560d4c452d9da0c07e692fdbfe6e2c81a2a91f2" From 014a4cd467160dc57ff5558fe80b4c4ba0b4e436 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 12:22:53 +0200 Subject: [PATCH 15/24] Undo types --- packages/server/tsconfig.build.json | 2 +- packages/server/tsconfig.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/tsconfig.build.json b/packages/server/tsconfig.build.json index bbce6a8bda..355b7ed6da 100644 --- a/packages/server/tsconfig.build.json +++ b/packages/server/tsconfig.build.json @@ -8,7 +8,7 @@ "esModuleInterop": true, "resolveJsonModule": true, "incremental": true, - "types": ["node"], + "types": ["node", "jest"], "outDir": "dist/src", "skipLibCheck": true, "baseUrl": ".", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 849e1e1a83..6e88b99c34 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -5,8 +5,7 @@ "declaration": true, "sourceMap": true, "baseUrl": ".", - "outDir": "dist", - "types": ["node", "jest"] + "outDir": "dist" }, "ts-node": { "require": ["tsconfig-paths/register"], From ce3f8853910c53fa57e2978a3c12fdc3109590c9 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 11:00:16 +0000 Subject: [PATCH 16/24] Bump version to 2.6.19-alpha.8 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1f656be795..99de26f145 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.7", + "version": "2.6.19-alpha.8", "npmClient": "yarn", "packages": [ "packages/backend-core", From 73f99b99d2cece375528234d92d487bfb8dcd782 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 14:25:50 +0200 Subject: [PATCH 17/24] Adding conflicting externals --- packages/worker/project.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/project.json b/packages/worker/project.json index 80b4a1d7c6..65f9b2d567 100644 --- a/packages/worker/project.json +++ b/packages/worker/project.json @@ -17,6 +17,7 @@ "output": "." } ], + "external": ["graphql/*", "deasync", "mock-aws-s3", "nock"], "format": ["cjs"], "esbuildOptions": { "outExtension": { From 9519ff6535cc5bc077193ad54166bb4bde05cb9f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 14:27:18 +0200 Subject: [PATCH 18/24] Froze lockfile on release --- .github/workflows/release-develop.yml | 2 +- .github/workflows/release-master.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 6618884369..c724b717e2 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -37,7 +37,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 6b2aecbf7d..4959194064 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ From b6ecf085875ab7b44a6951e2484fcf3b6c25088a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 13:14:47 +0000 Subject: [PATCH 19/24] Bump version to 2.6.19-alpha.9 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 99de26f145..d6ac7aac7c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.8", + "version": "2.6.19-alpha.9", "npmClient": "yarn", "packages": [ "packages/backend-core", From 82ac46e5ebb95c49223972aabd2e8ed4c4285af8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 13:23:45 +0000 Subject: [PATCH 20/24] Bump version to 2.6.19-alpha.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d6ac7aac7c..1d4b21537c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.9", + "version": "2.6.19-alpha.10", "npmClient": "yarn", "packages": [ "packages/backend-core", From 146940f6aef4a7c9e01f5e52dcf0b63e86bedc1d Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 25 May 2023 17:50:52 +0100 Subject: [PATCH 21/24] Added error handling for export row front end and fixed row export backend behaviour --- .../DataTable/modals/ExportModal.svelte | 31 ++++++++++++------- .../src/api/controllers/row/internal.ts | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte index 68968d5785..6cefe244bf 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte @@ -113,17 +113,26 @@ }) download(data, `export.${exportFormat}`) } else if (filters || sorting) { - const data = await API.exportRows({ - tableId: view, - format: exportFormat, - search: { - query: luceneFilter, - sort: sorting?.sortColumn, - sortOrder: sorting?.sortOrder, - paginate: false, - }, - }) - download(data, `export.${exportFormat}`) + let response + try { + response = await API.exportRows({ + tableId: view, + format: exportFormat, + search: { + query: luceneFilter, + sort: sorting?.sortColumn, + sortOrder: sorting?.sortOrder, + paginate: false, + }, + }) + } catch (e) { + console.error("Failed to export", e) + notifications.error("Export Failed") + } + if (response) { + download(response, `export.${exportFormat}`) + notifications.success("Export Successful") + } } else { await exportView() } diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index e3cb419236..87d8cd7e9a 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -415,7 +415,7 @@ export async function exportRows(ctx: UserCtx) { result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await exports.search(ctx) + let searchResponse = await search(ctx) result = searchResponse.rows } From b5c98871adeb7af58ba35d58a3be92aaab306526 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 25 May 2023 18:05:07 +0100 Subject: [PATCH 22/24] Update table data via CSV import (#10313) * Add identifierFields select for import * Update rows on import (Internal DB) * Only allow internal DB to upsert import CSV * Clear identifierFields when turning off update * Passing table instead of tableId * Pass table * Pass tableType --- .../backend/DataTable/DataTable.svelte | 1 + .../DataTable/buttons/ImportButton.svelte | 3 +- .../buttons/grid/GridImportButton.svelte | 3 +- .../DataTable/modals/ImportModal.svelte | 12 ++++++- .../ExistingTableDataImport.svelte | 21 +++++++++++- packages/frontend-core/src/api/tables.js | 6 ++-- .../src/components/grid/layout/Grid.svelte | 2 ++ .../src/api/controllers/table/internal.ts | 8 ++--- .../server/src/api/controllers/table/utils.ts | 33 ++++++++++++++++++- 9 files changed, 76 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index dfe30a3711..83e115ebf2 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -32,6 +32,7 @@ - + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte index 5aa530b028..5b89d92438 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte @@ -4,11 +4,12 @@ export let disabled = false - const { rows, tableId } = getContext("grid") + const { rows, tableId, tableType } = getContext("grid") diff --git a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte index 173fb88829..c020d1a7ac 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte @@ -13,15 +13,18 @@ const dispatch = createEventDispatcher() export let tableId + export let tableType let rows = [] let allValid = false let displayColumn = null + let identifierFields = [] async function importData() { try { await API.importTableData({ tableId, rows, + identifierFields, }) notifications.success("Rows successfully imported") } catch (error) { @@ -45,6 +48,13 @@ - + diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index cb150f71fc..eba17e8ab9 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -1,5 +1,5 @@