From 84f52683b23f4f3d3078094b9de43f7e239b85a6 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:55:44 +0100 Subject: [PATCH 01/21] chatgpt automation block --- packages/server/package.json | 1 + packages/server/src/automations/actions.ts | 3 + .../server/src/automations/steps/openai.ts | 104 ++++++++++++++++++ .../src/automations/tests/openai.spec.ts | 85 ++++++++++++++ packages/server/src/environment.ts | 1 + .../types/src/documents/app/automation.ts | 1 + yarn.lock | 20 +++- 7 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 packages/server/src/automations/steps/openai.ts create mode 100644 packages/server/src/automations/tests/openai.spec.ts diff --git a/packages/server/package.json b/packages/server/package.json index 6aadfd15a0..2e53c0e7ac 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -99,6 +99,7 @@ "mysql2": "2.3.3", "node-fetch": "2.6.7", "open": "8.4.0", + "openai": "^3.2.1", "pg": "8.5.1", "posthog-node": "1.3.0", "pouchdb": "7.3.0", diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 2a6b760725..de92efb676 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -14,6 +14,7 @@ import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" +import * as openai from "./steps/openai" import env from "../environment" import { AutomationStepSchema, @@ -39,6 +40,7 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, + OPEN_AI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -59,6 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, + OPEN_AI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts new file mode 100644 index 0000000000..79586bb712 --- /dev/null +++ b/packages/server/src/automations/steps/openai.ts @@ -0,0 +1,104 @@ +import { Configuration, OpenAIApi } from "openai"; +import { + AutomationActionStepId, + AutomationStepSchema, + AutomationStepInput, + AutomationStepType, + AutomationIOType, +} from "@budibase/types" +import * as automationUtils from "../automationUtils" +import environment from "../../environment"; + +enum Model { + GPT_35_TURBO = "gpt-3.5-turbo", + // will only work with api keys that have access to the GPT4 API + // GPT_4 = "gpt-4", +} + +export const definition: AutomationStepSchema = { + name: "OpenAI", + tagline: "Send prompts to ChatGPT", + icon: "Algorithm", + description: "Interact with the OpenAI ChatGPT API.", + type: AutomationStepType.ACTION, + internal: true, + stepId: AutomationActionStepId.OPEN_AI, + inputs: { + prompt: "", + }, + schema: { + inputs: { + properties: { + prompt: { + type: AutomationIOType.STRING, + title: "Prompt", + }, + model: { + type: AutomationIOType.STRING, + title: "Model", + enum: Object.values(Model), + }, + }, + required: ["prompt", "model"], + }, + outputs: { + properties: { + success: { + type: AutomationIOType.BOOLEAN, + description: "Whether the action was successful", + }, + response: { + type: AutomationIOType.STRING, + description: "What was output", + }, + }, + required: ["success", "response"], + }, + }, +} + +export async function run({ inputs, context }: AutomationStepInput) { + if (!environment.OPENAI_API_KEY) { + return { + success: false, + response: "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + + if (inputs.prompt == null) { + return { + success: false, + response: "Budibase OpenAI Automation Failed: No prompt supplied", + } + } + + try { + const configuration = new Configuration({ + apiKey: environment.OPENAI_API_KEY, + }); + + const openai = new OpenAIApi(configuration); + + const completion = await openai.createChatCompletion({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt + } + ], + }); + + let response = completion?.data?.choices[0]?.message?.content + + return { + response, + success: true, + } + } catch (err) { + return { + success: false, + response: automationUtils.getError(err), + } + } +} diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts new file mode 100644 index 0000000000..3ba9463f21 --- /dev/null +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -0,0 +1,85 @@ +const setup = require("./utilities") +import environment from "../../environment"; +import openai from "openai" + +jest.mock("openai", jest.fn(() => ({ + Configuration: jest.fn(), + OpenAIApi: jest.fn(() => ({ + createChatCompletion: jest.fn(() => ({ + data: { + choices: [ + { + message: { + content: "This is a test" + }, + } + ] + } + })) + })) +}))) + +const OPENAI_PROMPT = "What is the meaning of life?" + +describe("test the openai action", () => { + let config = setup.getConfig() + + beforeAll(async () => { + await config.init() + }) + + beforeEach(() => { + environment.OPENAI_API_KEY = "abc123" + }) + + afterAll(setup.afterAll) + + + it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { + delete environment.OPENAI_API_KEY + + let res = await setup.runStep("OPEN_AI", + { + prompt: OPENAI_PROMPT + } + ) + expect(res.response).toEqual("OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.") + expect(res.success).toBeFalsy() + }) + + it("should be able to receive a response from ChatGPT given a prompt", async () => { + const res = await setup.runStep("OPEN_AI", + { + prompt: OPENAI_PROMPT + } + ) + expect(res.response).toEqual("This is a test") + expect(res.success).toBeTruthy() + }) + + + it("should present the correct error message when a prompt is not provided", async () => { + const res = await setup.runStep("OPEN_AI", + { + prompt: null + } + ) + expect(res.response).toEqual("Budibase OpenAI Automation Failed: No prompt supplied") + expect(res.success).toBeFalsy() + }) + + it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { + openai.OpenAIApi.mockImplementation(() => ({ + createChatCompletion: jest.fn(() => { + throw new Error("An error occurred while calling createChatCompletion"); + }), + })); + + const res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }); + + expect(res.response).toEqual("Error: An error occurred while calling createChatCompletion") + expect(res.success).toBeFalsy() + }); +}) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 1bd5a6486c..9a52b18e08 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -72,6 +72,7 @@ const environment = { BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", + OPENAI_API_KEY: process.env.OPENAI_API_KEY, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index aa600c6375..eaff533761 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -56,6 +56,7 @@ export enum AutomationActionStepId { FILTER = "FILTER", QUERY_ROWS = "QUERY_ROWS", LOOP = "LOOP", + OPEN_AI = "OPEN_AI", // these used to be lowercase step IDs, maintain for backwards compat discord = "discord", slack = "slack", diff --git a/yarn.lock b/yarn.lock index 260f0ae6a6..3d233dbe76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.5.6-alpha.29": - version "2.5.6-alpha.29" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.29.tgz#71414f68a296535ef53ffb0453352ea137c4aeab" - integrity sha512-tQuzMOo2WFxKvsUgYAfUEcLabRpmAD7hPlhBhCFzYasaXNbJiPhcwv4i52US0i0Wr2IXMb2X0d7fwa8tnbKzIA== +"@budibase/pro@2.5.6-alpha.30": + version "2.5.6-alpha.30" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.5.6-alpha.30.tgz#9b8089a983fd61a062f31a8e5757d7bb5b56fb8c" + integrity sha512-YTyjMHK/wsSOFJkON7a5WRJSgAr8Gh/cflRzifm6Jw1Gb8S8B8Z6uTWW/S7+psVBRGeUfV1s8biYNr71tXz2Ng== dependencies: - "@budibase/backend-core" "2.5.6-alpha.29" + "@budibase/backend-core" "2.5.6-alpha.30" "@budibase/shared-core" "2.4.44-alpha.1" "@budibase/string-templates" "2.4.44-alpha.1" - "@budibase/types" "2.5.6-alpha.29" + "@budibase/types" "2.5.6-alpha.30" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -18358,6 +18358,14 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" +openai@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/openai/-/openai-3.2.1.tgz#1fa35bdf979cbde8453b43f2dd3a7d401ee40866" + integrity sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A== + dependencies: + axios "^0.26.0" + form-data "^4.0.0" + openapi-response-validator@^9.2.0: version "9.3.1" resolved "https://registry.yarnpkg.com/openapi-response-validator/-/openapi-response-validator-9.3.1.tgz#54284d8be608ef53283cbe7448accce8106b1c56" From 4f020a4db4766974d092827e8c843c2a79050fb1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:56:46 +0100 Subject: [PATCH 02/21] lint --- .../server/src/automations/steps/openai.ts | 17 +++-- .../src/automations/tests/openai.spec.ts | 75 ++++++++++--------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 79586bb712..a7c22ffd0c 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -1,4 +1,4 @@ -import { Configuration, OpenAIApi } from "openai"; +import { Configuration, OpenAIApi } from "openai" import { AutomationActionStepId, AutomationStepSchema, @@ -7,7 +7,7 @@ import { AutomationIOType, } from "@budibase/types" import * as automationUtils from "../automationUtils" -import environment from "../../environment"; +import environment from "../../environment" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", @@ -61,7 +61,8 @@ export async function run({ inputs, context }: AutomationStepInput) { if (!environment.OPENAI_API_KEY) { return { success: false, - response: "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", } } @@ -75,19 +76,19 @@ export async function run({ inputs, context }: AutomationStepInput) { try { const configuration = new Configuration({ apiKey: environment.OPENAI_API_KEY, - }); + }) - const openai = new OpenAIApi(configuration); + const openai = new OpenAIApi(configuration) const completion = await openai.createChatCompletion({ model: inputs.model, messages: [ { role: "user", - content: inputs.prompt - } + content: inputs.prompt, + }, ], - }); + }) let response = completion?.data?.choices[0]?.message?.content diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3ba9463f21..032f670db1 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -1,23 +1,26 @@ const setup = require("./utilities") -import environment from "../../environment"; +import environment from "../../environment" import openai from "openai" -jest.mock("openai", jest.fn(() => ({ - Configuration: jest.fn(), - OpenAIApi: jest.fn(() => ({ - createChatCompletion: jest.fn(() => ({ - data: { - choices: [ - { - message: { - content: "This is a test" +jest.mock( + "openai", + jest.fn(() => ({ + Configuration: jest.fn(), + OpenAIApi: jest.fn(() => ({ + createChatCompletion: jest.fn(() => ({ + data: { + choices: [ + { + message: { + content: "This is a test", + }, }, - } - ] - } - })) + ], + }, + })), + })), })) -}))) +) const OPENAI_PROMPT = "What is the meaning of life?" @@ -34,52 +37,50 @@ describe("test the openai action", () => { afterAll(setup.afterAll) - it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { delete environment.OPENAI_API_KEY - let res = await setup.runStep("OPEN_AI", - { - prompt: OPENAI_PROMPT - } + let res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }) + expect(res.response).toEqual( + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable." ) - expect(res.response).toEqual("OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.") expect(res.success).toBeFalsy() }) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await setup.runStep("OPEN_AI", - { - prompt: OPENAI_PROMPT - } - ) + const res = await setup.runStep("OPEN_AI", { + prompt: OPENAI_PROMPT, + }) expect(res.response).toEqual("This is a test") expect(res.success).toBeTruthy() }) - it("should present the correct error message when a prompt is not provided", async () => { - const res = await setup.runStep("OPEN_AI", - { - prompt: null - } + const res = await setup.runStep("OPEN_AI", { + prompt: null, + }) + expect(res.response).toEqual( + "Budibase OpenAI Automation Failed: No prompt supplied" ) - expect(res.response).toEqual("Budibase OpenAI Automation Failed: No prompt supplied") expect(res.success).toBeFalsy() }) it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { openai.OpenAIApi.mockImplementation(() => ({ createChatCompletion: jest.fn(() => { - throw new Error("An error occurred while calling createChatCompletion"); + throw new Error("An error occurred while calling createChatCompletion") }), - })); + })) const res = await setup.runStep("OPEN_AI", { prompt: OPENAI_PROMPT, - }); + }) - expect(res.response).toEqual("Error: An error occurred while calling createChatCompletion") + expect(res.response).toEqual( + "Error: An error occurred while calling createChatCompletion" + ) expect(res.success).toBeFalsy() - }); + }) }) From e70e3ae662fee5fa11e1123bf787640f9e750627 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 26 Apr 2023 15:58:21 +0100 Subject: [PATCH 03/21] rename --- packages/server/src/automations/actions.ts | 4 ++-- packages/server/src/automations/steps/openai.ts | 4 ++-- packages/server/src/automations/tests/openai.spec.ts | 8 ++++---- packages/types/src/documents/app/automation.ts | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index de92efb676..f0feab006c 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -40,7 +40,7 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, - OPEN_AI: openai.run, + OPENAI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -61,7 +61,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, - OPEN_AI: openai.definition, + OPENAI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index a7c22ffd0c..265a40d466 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -22,7 +22,7 @@ export const definition: AutomationStepSchema = { description: "Interact with the OpenAI ChatGPT API.", type: AutomationStepType.ACTION, internal: true, - stepId: AutomationActionStepId.OPEN_AI, + stepId: AutomationActionStepId.OPENAI, inputs: { prompt: "", }, @@ -90,7 +90,7 @@ export async function run({ inputs, context }: AutomationStepInput) { ], }) - let response = completion?.data?.choices[0]?.message?.content + const response = completion?.data?.choices[0]?.message?.content return { response, diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 032f670db1..31f7e48305 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -40,7 +40,7 @@ describe("test the openai action", () => { it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => { delete environment.OPENAI_API_KEY - let res = await setup.runStep("OPEN_AI", { + let res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) expect(res.response).toEqual( @@ -50,7 +50,7 @@ describe("test the openai action", () => { }) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) expect(res.response).toEqual("This is a test") @@ -58,7 +58,7 @@ describe("test the openai action", () => { }) it("should present the correct error message when a prompt is not provided", async () => { - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: null, }) expect(res.response).toEqual( @@ -74,7 +74,7 @@ describe("test the openai action", () => { }), })) - const res = await setup.runStep("OPEN_AI", { + const res = await setup.runStep("OPENAI", { prompt: OPENAI_PROMPT, }) diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index eaff533761..11e868023f 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -56,7 +56,7 @@ export enum AutomationActionStepId { FILTER = "FILTER", QUERY_ROWS = "QUERY_ROWS", LOOP = "LOOP", - OPEN_AI = "OPEN_AI", + OPENAI = "OPENAI", // these used to be lowercase step IDs, maintain for backwards compat discord = "discord", slack = "slack", From db5d051755bf1017b093a4147a4d62109870b370 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 May 2023 15:26:27 +0100 Subject: [PATCH 04/21] GPT4 support --- packages/server/src/automations/steps/openai.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 265a40d466..88d3fb8b85 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -12,7 +12,7 @@ import environment from "../../environment" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API - // GPT_4 = "gpt-4", + GPT_4 = "gpt-4", } export const definition: AutomationStepSchema = { From 90ffa338c3ecb3489496d2182d3e2940ccb88b7f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 May 2023 16:23:34 +0100 Subject: [PATCH 05/21] Fixes the single image build, as well as fixing the nodemon process for the server. --- hosting/single/Dockerfile | 12 +++++------- package.json | 8 ++++---- packages/server/nodemon.json | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index fadcf235e9..985570e4e1 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,22 +1,22 @@ -FROM node:14-slim as build +FROM node:16-slim as build # install node-gyp dependencies RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python # add pin script WORKDIR / -ADD scripts/pinVersions.js scripts/cleanup.sh ./ +ADD scripts/cleanup.sh ./ RUN chmod +x /cleanup.sh # build server WORKDIR /app ADD packages/server . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn && /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn && /cleanup.sh FROM budibase/couchdb ARG TARGETARCH @@ -31,9 +31,7 @@ COPY --from=build /worker /worker # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \ - apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ - apt-get update + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs diff --git a/package.json b/package.json index 70eceef4fd..8052597ce3 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "scripts": { "preinstall": "node scripts/syncProPackage.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "./scripts/bootstrap.sh && ./scripts/link-dependencies.sh", + "bootstrap": "yarn && ./scripts/link-dependencies.sh", "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", @@ -60,16 +60,16 @@ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", - "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:pre": "lerna run --stream build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", + "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", + "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index aa72d11777..77375546e1 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -6,5 +6,5 @@ "src/**/*.spec.js", "../backend-core/dist/**/*" ], - "exec": "node ../../scripts/build && node ./dist/index.js" + "exec": "node ./scripts/build.js && node ./dist/index.js" } From 40a3624c5b9d053e875f3cf166d13b2095d7ea10 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 May 2023 16:35:05 +0100 Subject: [PATCH 06/21] Removing directions towards bootstrap command as it is no longer required. --- docs/CONTRIBUTING.md | 4 +--- package.json | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6e667d23a8..ac35929be1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running ( `yarn` to install project dependencies -`yarn bootstrap` will install all budibase modules and symlink them together using lerna. - `yarn build` will build all budibase packages. #### 4. Running @@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README. Note that only budibase maintainers will be able to access the pro repo. -The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. +By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. ### Troubleshooting diff --git a/package.json b/package.json index 8052597ce3..277b213b78 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,8 @@ }, "scripts": { "preinstall": "node scripts/syncProPackage.js", - "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "yarn && ./scripts/link-dependencies.sh", + "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", + "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", "build": "yarn nx run-many -t=build", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", From 733e724eb309163c6577674b2e0e0051c3ef369e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 24 May 2023 16:38:24 +0100 Subject: [PATCH 07/21] PR comments. --- hosting/single/Dockerfile | 4 ++-- packages/server/Dockerfile | 2 +- packages/worker/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 985570e4e1..64a6b01365 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -11,12 +11,12 @@ RUN chmod +x /cleanup.sh # build server WORKDIR /app ADD packages/server . -RUN yarn && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . -RUN yarn && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh FROM budibase/couchdb ARG TARGETARCH diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index bf9138db86..5880a84326 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -27,7 +27,7 @@ COPY scripts/integrations/oracle/ scripts/integrations/oracle/ RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh COPY package.json . -RUN yarn +RUN yarn install --frozen-lockfile --production=true # Remove unneeded data from file system to reduce image size RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 16fde4edc1..7b8f991013 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -13,7 +13,7 @@ RUN yarn global add pm2 COPY dist/package.json . -RUN yarn +RUN yarn install --frozen-lockfile --production=true # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ && yarn cache clean From 8c6efb8ebbbf21d236182db3e49209a3ee2cb060 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 24 May 2023 16:01:26 +0000 Subject: [PATCH 08/21] Bump version to 2.6.19-alpha.7 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1362495320..1f656be795 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.6", + "version": "2.6.19-alpha.7", "npmClient": "yarn", "packages": [ "packages/backend-core", From f86d321e2ea7978057f4e2231949dd3bc96c4f36 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 May 2023 17:17:23 +0100 Subject: [PATCH 09/21] restrict openai just to self host --- packages/server/src/automations/actions.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 01a9731d20..2f64e75816 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -14,7 +14,6 @@ import * as filter from "./steps/filter" import * as delay from "./steps/delay" import * as queryRow from "./steps/queryRows" import * as loop from "./steps/loop" -import * as openai from "./steps/openai" import env from "../environment" import { AutomationStepSchema, @@ -40,7 +39,6 @@ const ACTION_IMPLS: Record< DELAY: delay.run, FILTER: filter.run, QUERY_ROWS: queryRow.run, - OPENAI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -61,7 +59,6 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = FILTER: filter.definition, QUERY_ROWS: queryRow.definition, LOOP: loop.definition, - OPENAI: openai.definition, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.definition, slack: slack.definition, @@ -74,10 +71,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record = // ran at all if (env.SELF_HOSTED) { const bash = require("./steps/bash") + const openai = require("./steps/openai") + // @ts-ignore ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + + ACTION_IMPLS.OPENAI = openai.run + BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition } export async function getActionDefinitions() { From 429dc8f6d2b52350e64cfb14401bfbb96d20e23f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 11:49:18 +0200 Subject: [PATCH 10/21] Tidy up tsconfigs --- packages/server/tsconfig.build.json | 2 +- packages/server/tsconfig.json | 8 +------- packages/worker/tsconfig.json | 8 +------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/server/tsconfig.build.json b/packages/server/tsconfig.build.json index 355b7ed6da..bbce6a8bda 100644 --- a/packages/server/tsconfig.build.json +++ b/packages/server/tsconfig.build.json @@ -8,7 +8,7 @@ "esModuleInterop": true, "resolveJsonModule": true, "incremental": true, - "types": ["node", "jest"], + "types": ["node"], "outDir": "dist/src", "skipLibCheck": true, "baseUrl": ".", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index cf86094cd2..849e1e1a83 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -6,13 +6,7 @@ "sourceMap": true, "baseUrl": ".", "outDir": "dist", - "paths": { - "@budibase/types": ["../types/src"], - "@budibase/backend-core": ["../backend-core/src"], - "@budibase/backend-core/*": ["../backend-core/*"], - "@budibase/shared-core": ["../shared-core/src"], - "@budibase/pro": ["../pro/packages/pro/src"] - } + "types": ["node", "jest"] }, "ts-node": { "require": ["tsconfig-paths/register"], diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json index 61f3f9f5e1..147ef1f700 100644 --- a/packages/worker/tsconfig.json +++ b/packages/worker/tsconfig.json @@ -4,13 +4,7 @@ "composite": true, "declaration": true, "sourceMap": true, - "baseUrl": ".", - "paths": { - "@budibase/types": ["../types/src"], - "@budibase/backend-core": ["../backend-core/src"], - "@budibase/backend-core/*": ["../backend-core/*"], - "@budibase/pro": ["../pro/packages/pro/src"] - } + "baseUrl": "." }, "ts-node": { "require": ["tsconfig-paths/register"], From bdda73a70bb91e42babd81ee5eeccfd38d0072b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 11:49:38 +0200 Subject: [PATCH 11/21] Respect tsconfig paths --- package.json | 1 + scripts/build.js | 4 ++++ yarn.lock | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 277b213b78..edbb82b892 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "private": true, "devDependencies": { "@esbuild-plugins/node-resolve": "^0.2.2", + "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@nx/esbuild": "16.2.1", "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", diff --git a/scripts/build.js b/scripts/build.js index 986dbad13f..34f0a70d35 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -9,6 +9,9 @@ const path = require("path") const { build } = require("esbuild") const { default: NodeResolve } = require("@esbuild-plugins/node-resolve") +const { + default: TsconfigPathsPlugin, +} = require("@esbuild-plugins/tsconfig-paths") var argv = require("minimist")(process.argv.slice(2)) @@ -23,6 +26,7 @@ function runBuild(entry, outfile) { sourcemap: isDev, tsconfig, plugins: [ + TsconfigPathsPlugin({ tsconfig }), NodeResolve({ extensions: [".ts", ".js"], onResolved: resolved => { diff --git a/yarn.lock b/yarn.lock index 086342b0f2..e044074a50 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2047,6 +2047,15 @@ escape-string-regexp "^4.0.0" resolve "^1.19.0" +"@esbuild-plugins/tsconfig-paths@^0.1.2": + version "0.1.2" + resolved "https://registry.yarnpkg.com/@esbuild-plugins/tsconfig-paths/-/tsconfig-paths-0.1.2.tgz#1955de0a124ecf4364717a2fadbfbea876955232" + integrity sha512-TusFR26Y+Ze+Zm+NdfqZTSG4XyrXKxIaAfYCL3jASEI/gHjSdoCujATjzNWaaXs6Sk6Bv2D7NLr4Jdz1gysy/Q== + dependencies: + debug "^4.3.1" + find-up "^5.0.0" + strip-json-comments "^3.1.1" + "@esbuild/android-arm64@0.16.17": version "0.16.17" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.16.17.tgz#cf91e86df127aa3d141744edafcba0abdc577d23" @@ -12085,6 +12094,14 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + findit2@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6" @@ -17012,6 +17029,13 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + lodash-es@^4.17.11: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" @@ -19371,7 +19395,7 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" -p-limit@^3.1.0: +p-limit@^3.0.2, p-limit@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== @@ -19406,6 +19430,13 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map-series@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map-series/-/p-map-series-2.1.0.tgz#7560d4c452d9da0c07e692fdbfe6e2c81a2a91f2" From 014a4cd467160dc57ff5558fe80b4c4ba0b4e436 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 12:22:53 +0200 Subject: [PATCH 12/21] Undo types --- packages/server/tsconfig.build.json | 2 +- packages/server/tsconfig.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/server/tsconfig.build.json b/packages/server/tsconfig.build.json index bbce6a8bda..355b7ed6da 100644 --- a/packages/server/tsconfig.build.json +++ b/packages/server/tsconfig.build.json @@ -8,7 +8,7 @@ "esModuleInterop": true, "resolveJsonModule": true, "incremental": true, - "types": ["node"], + "types": ["node", "jest"], "outDir": "dist/src", "skipLibCheck": true, "baseUrl": ".", diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 849e1e1a83..6e88b99c34 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -5,8 +5,7 @@ "declaration": true, "sourceMap": true, "baseUrl": ".", - "outDir": "dist", - "types": ["node", "jest"] + "outDir": "dist" }, "ts-node": { "require": ["tsconfig-paths/register"], From ce3f8853910c53fa57e2978a3c12fdc3109590c9 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 11:00:16 +0000 Subject: [PATCH 13/21] Bump version to 2.6.19-alpha.8 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 1f656be795..99de26f145 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.7", + "version": "2.6.19-alpha.8", "npmClient": "yarn", "packages": [ "packages/backend-core", From 73f99b99d2cece375528234d92d487bfb8dcd782 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 14:25:50 +0200 Subject: [PATCH 14/21] Adding conflicting externals --- packages/worker/project.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/project.json b/packages/worker/project.json index 80b4a1d7c6..65f9b2d567 100644 --- a/packages/worker/project.json +++ b/packages/worker/project.json @@ -17,6 +17,7 @@ "output": "." } ], + "external": ["graphql/*", "deasync", "mock-aws-s3", "nock"], "format": ["cjs"], "esbuildOptions": { "outExtension": { From 9519ff6535cc5bc077193ad54166bb4bde05cb9f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 25 May 2023 14:27:18 +0200 Subject: [PATCH 15/21] Froze lockfile on release --- .github/workflows/release-develop.yml | 2 +- .github/workflows/release-master.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 6618884369..c724b717e2 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -37,7 +37,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 6b2aecbf7d..4959194064 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -42,7 +42,7 @@ jobs: with: node-version: 14.x - - run: yarn + - run: yarn install --frozen-lockfile - name: Update versions run: | version=$(cat lerna.json \ From b6ecf085875ab7b44a6951e2484fcf3b6c25088a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 13:14:47 +0000 Subject: [PATCH 16/21] Bump version to 2.6.19-alpha.9 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 99de26f145..d6ac7aac7c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.8", + "version": "2.6.19-alpha.9", "npmClient": "yarn", "packages": [ "packages/backend-core", From 82ac46e5ebb95c49223972aabd2e8ed4c4285af8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 25 May 2023 13:23:45 +0000 Subject: [PATCH 17/21] Bump version to 2.6.19-alpha.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d6ac7aac7c..1d4b21537c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.19-alpha.9", + "version": "2.6.19-alpha.10", "npmClient": "yarn", "packages": [ "packages/backend-core", From 146940f6aef4a7c9e01f5e52dcf0b63e86bedc1d Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 25 May 2023 17:50:52 +0100 Subject: [PATCH 18/21] Added error handling for export row front end and fixed row export backend behaviour --- .../DataTable/modals/ExportModal.svelte | 31 ++++++++++++------- .../src/api/controllers/row/internal.ts | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte index 68968d5785..6cefe244bf 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte @@ -113,17 +113,26 @@ }) download(data, `export.${exportFormat}`) } else if (filters || sorting) { - const data = await API.exportRows({ - tableId: view, - format: exportFormat, - search: { - query: luceneFilter, - sort: sorting?.sortColumn, - sortOrder: sorting?.sortOrder, - paginate: false, - }, - }) - download(data, `export.${exportFormat}`) + let response + try { + response = await API.exportRows({ + tableId: view, + format: exportFormat, + search: { + query: luceneFilter, + sort: sorting?.sortColumn, + sortOrder: sorting?.sortOrder, + paginate: false, + }, + }) + } catch (e) { + console.error("Failed to export", e) + notifications.error("Export Failed") + } + if (response) { + download(response, `export.${exportFormat}`) + notifications.success("Export Successful") + } } else { await exportView() } diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index e3cb419236..87d8cd7e9a 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -415,7 +415,7 @@ export async function exportRows(ctx: UserCtx) { result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await exports.search(ctx) + let searchResponse = await search(ctx) result = searchResponse.rows } From b5c98871adeb7af58ba35d58a3be92aaab306526 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 25 May 2023 18:05:07 +0100 Subject: [PATCH 19/21] Update table data via CSV import (#10313) * Add identifierFields select for import * Update rows on import (Internal DB) * Only allow internal DB to upsert import CSV * Clear identifierFields when turning off update * Passing table instead of tableId * Pass table * Pass tableType --- .../backend/DataTable/DataTable.svelte | 1 + .../DataTable/buttons/ImportButton.svelte | 3 +- .../buttons/grid/GridImportButton.svelte | 3 +- .../DataTable/modals/ImportModal.svelte | 12 ++++++- .../ExistingTableDataImport.svelte | 21 +++++++++++- packages/frontend-core/src/api/tables.js | 6 ++-- .../src/components/grid/layout/Grid.svelte | 2 ++ .../src/api/controllers/table/internal.ts | 8 ++--- .../server/src/api/controllers/table/utils.ts | 33 ++++++++++++++++++- 9 files changed, 76 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/DataTable.svelte b/packages/builder/src/components/backend/DataTable/DataTable.svelte index dfe30a3711..83e115ebf2 100644 --- a/packages/builder/src/components/backend/DataTable/DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/DataTable.svelte @@ -32,6 +32,7 @@ - + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte index 5aa530b028..5b89d92438 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte @@ -4,11 +4,12 @@ export let disabled = false - const { rows, tableId } = getContext("grid") + const { rows, tableId, tableType } = getContext("grid") diff --git a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte index 173fb88829..c020d1a7ac 100644 --- a/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte @@ -13,15 +13,18 @@ const dispatch = createEventDispatcher() export let tableId + export let tableType let rows = [] let allValid = false let displayColumn = null + let identifierFields = [] async function importData() { try { await API.importTableData({ tableId, rows, + identifierFields, }) notifications.success("Rows successfully imported") } catch (error) { @@ -45,6 +48,13 @@ - + diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index cb150f71fc..eba17e8ab9 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -1,5 +1,5 @@