Merge remote-tracking branch 'origin/develop' into feature/sync-automations

.github/workflows/release-develop.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           node-version: 14.x
 
-      - run: yarn
+      - run: yarn install --frozen-lockfile
       - name: Update versions
         run: |
           version=$(cat lerna.json \

.github/workflows/release-master.yml

@@ -42,7 +42,7 @@ jobs:
         with:
           node-version: 14.x
 
-      - run: yarn
+      - run: yarn install --frozen-lockfile
       - name: Update versions
         run: |
           version=$(cat lerna.json \

docs/CONTRIBUTING.md

@@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running (
 
 `yarn` to install project dependencies
 
-`yarn bootstrap` will install all budibase modules and symlink them together using lerna.
-
 `yarn build` will build all budibase packages.
 
 #### 4. Running
@@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README.
 
 Note that only budibase maintainers will be able to access the pro repo.
 
-The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
+By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
 
 ### Troubleshooting
 

hosting/single/Dockerfile

@@ -1,22 +1,22 @@
-FROM node:14-slim as build
+FROM node:16-slim as build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python
 
 # add pin script
 WORKDIR /
-ADD scripts/pinVersions.js scripts/cleanup.sh ./
+ADD scripts/cleanup.sh ./
 RUN chmod +x /cleanup.sh
 
 # build server
 WORKDIR /app
 ADD packages/server .
-RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
 
 # build worker
 WORKDIR /worker
 ADD packages/worker .
-RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
+RUN yarn install --frozen-lockfile --production=true && /cleanup.sh
 
 FROM budibase/couchdb
 ARG TARGETARCH
@@ -31,9 +31,7 @@ COPY --from=build /worker /worker
 
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server
-  apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \
-  apt-get update
 
 # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx
 WORKDIR /nodejs

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.6",
+  "version": "2.6.19-alpha.12",
   "npmClient": "yarn",
   "packages": [
     "packages/backend-core",

package.json

@@ -3,6 +3,7 @@
   "private": true,
   "devDependencies": {
     "@esbuild-plugins/node-resolve": "^0.2.2",
+    "@esbuild-plugins/tsconfig-paths": "^0.1.2",
     "@nx/esbuild": "16.2.1",
     "@nx/js": "16.2.1",
     "@rollup/plugin-json": "^4.0.2",
@@ -29,8 +30,8 @@
   },
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
-    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
-    "bootstrap": "./scripts/bootstrap.sh  && ./scripts/link-dependencies.sh",
+    "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'",
     "build": "yarn nx run-many -t=build",
     "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
     "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
@@ -60,16 +61,16 @@
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
     "build:specs": "lerna run --stream specs",
-    "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:pre": "lerna run --stream build && lerna run --stream predocker",
     "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
     "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
     "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
     "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
-    "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image",
+    "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image",
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -32,6 +32,7 @@
   <Grid
     {API}
     tableId={id}
+    tableType={$tables.selected?.type}
     allowAddRows={!isUsersTable}
     allowDeleteRows={!isUsersTable}
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}

packages/builder/src/components/backend/DataTable/buttons/ImportButton.svelte

@@ -3,6 +3,7 @@
   import ImportModal from "../modals/ImportModal.svelte"
 
   export let tableId
+  export let tableType
   export let disabled
 
   let modal
@@ -12,5 +13,5 @@
   Import
 </ActionButton>
 <Modal bind:this={modal}>
-  <ImportModal {tableId} on:importrows />
+  <ImportModal {tableId} {tableType} on:importrows />
 </Modal>

packages/builder/src/components/backend/DataTable/buttons/grid/GridImportButton.svelte

@@ -4,11 +4,12 @@
 
   export let disabled = false
 
-  const { rows, tableId } = getContext("grid")
+  const { rows, tableId, tableType } = getContext("grid")
 </script>
 
 <ImportButton
   {disabled}
   tableId={$tableId}
+  {tableType}
   on:importrows={rows.actions.refreshData}
 />

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -113,17 +113,26 @@
       })
       download(data, `export.${exportFormat}`)
     } else if (filters || sorting) {
-      const data = await API.exportRows({
+      let response
-        tableId: view,
+      try {
-        format: exportFormat,
+        response = await API.exportRows({
-        search: {
+          tableId: view,
-          query: luceneFilter,
+          format: exportFormat,
-          sort: sorting?.sortColumn,
+          search: {
-          sortOrder: sorting?.sortOrder,
+            query: luceneFilter,
-          paginate: false,
+            sort: sorting?.sortColumn,
-        },
+            sortOrder: sorting?.sortOrder,
-      })
+            paginate: false,
-      download(data, `export.${exportFormat}`)
+          },
+        })
+      } catch (e) {
+        console.error("Failed to export", e)
+        notifications.error("Export Failed")
+      }
+      if (response) {
+        download(response, `export.${exportFormat}`)
+        notifications.success("Export Successful")
+      }
     } else {
       await exportView()
     }

packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

@@ -13,15 +13,18 @@
   const dispatch = createEventDispatcher()
 
   export let tableId
+  export let tableType
   let rows = []
   let allValid = false
   let displayColumn = null
+  let identifierFields = []
 
   async function importData() {
     try {
       await API.importTableData({
         tableId,
         rows,
+        identifierFields,
       })
       notifications.success("Rows successfully imported")
     } catch (error) {
@@ -45,6 +48,13 @@
   </Body>
   <Layout gap="XS" noPadding>
     <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport {tableId} bind:rows bind:allValid bind:displayColumn />
+    <TableDataImport
+      {tableId}
+      {tableType}
+      bind:rows
+      bind:allValid
+      bind:displayColumn
+      bind:identifierFields
+    />
   </Layout>
 </ModalContent>

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Toggle, Multiselect } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
@@ -9,14 +9,17 @@
   let fileType = null
 
   let loading = false
+  let updateExistingRows = false
   let validation = {}
   let validateHash = ""
   let schema = null
   let invalidColumns = []
 
   export let tableId = null
+  export let tableType
   export let rows = []
   export let allValid = false
+  export let identifierFields = []
 
   const typeOptions = [
     {
@@ -159,6 +162,22 @@
       </div>
     {/each}
   </div>
+  {#if tableType === "internal"}
+    <br />
+    <Toggle
+      bind:value={updateExistingRows}
+      on:change={() => (identifierFields = [])}
+      thin
+      text="Update existing rows"
+    />
+    {#if updateExistingRows}
+      <Multiselect
+        label="Identifier field(s)"
+        options={Object.keys(validation)}
+        bind:value={identifierFields}
+      />
+    {/if}
+  {/if}
   {#if invalidColumns.length > 0}
     <p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
       The following columns are present in the data you wish to import, but do

packages/frontend-core/src/api/tables.js

@@ -62,13 +62,15 @@ export const buildTableEndpoints = API => ({
   /**
    * Imports data into an existing table
    * @param tableId the table ID to import to
-   * @param data the data import object
+   * @param rows the data import object
+   * @param identifierFields column names to be used as keys for overwriting existing rows
    */
-  importTableData: async ({ tableId, rows }) => {
+  importTableData: async ({ tableId, rows, identifierFields }) => {
     return await API.post({
       url: `/api/tables/${tableId}/import`,
       body: {
         rows,
+        identifierFields,
       },
     })
   },

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -33,6 +33,7 @@
 
   export let API = null
   export let tableId = null
+  export let tableType = null
   export let schemaOverrides = null
   export let allowAddRows = true
   export let allowAddColumns = true
@@ -62,6 +63,7 @@
     rand,
     config,
     tableId: tableIdStore,
+    tableType,
     schemaOverrides: schemaOverridesStore,
   }
   context = { ...context, ...createEventManagers() }

packages/server/Dockerfile

@@ -27,7 +27,7 @@ COPY scripts/integrations/oracle/ scripts/integrations/oracle/
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
 
 COPY package.json .
-RUN yarn 
+RUN yarn install --frozen-lockfile --production=true
 # Remove unneeded data from file system to reduce image size
 RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
     && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp

packages/server/nodemon.json

@@ -6,5 +6,5 @@
     "src/**/*.spec.js",
     "../backend-core/dist/**/*"
   ],
-  "exec": "node ../../scripts/build && node ./dist/index.js"
+  "exec": "node ./scripts/build.js && node ./dist/index.js"
 }

packages/server/package.json

@@ -47,7 +47,7 @@
     "@apidevtools/swagger-parser": "10.0.3",
     "@budibase/backend-core": "0.0.1",
     "@budibase/client": "0.0.1",
-    "@budibase/pro": "0.0.1",
+    "@budibase/pro": "develop",
     "@budibase/shared-core": "0.0.1",
     "@budibase/string-templates": "0.0.1",
     "@budibase/types": "0.0.1",
@@ -99,6 +99,7 @@
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
     "open": "8.4.0",
+    "openai": "^3.2.1",
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",

packages/server/src/api/controllers/row/internal.ts

@@ -415,7 +415,7 @@ export async function exportRows(ctx: UserCtx) {
 
     result = await outputProcessing(table, response)
   } else if (query) {
-    let searchResponse = await exports.search(ctx)
+    let searchResponse = await search(ctx)
     result = searchResponse.rows
   }
 

packages/server/src/api/controllers/table/internal.ts

@@ -186,11 +186,7 @@ export async function destroy(ctx: any) {
 export async function bulkImport(ctx: any) {
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { rows } = ctx.request.body
+  const { rows, identifierFields } = ctx.request.body
-  await handleDataImport(ctx.user, table, rows)
+  await handleDataImport(ctx.user, table, rows, identifierFields)
-
-  // Ensure auto id and other table updates are persisted
-  await db.put(table)
-
   return table
 }

packages/server/src/api/controllers/table/utils.ts

@@ -149,7 +149,12 @@ export function importToRows(
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, rows: any) {
+export async function handleDataImport(
+  user: any,
+  table: any,
+  rows: any,
+  identifierFields: Array<string> = []
+) {
   const schema: unknown = table.schema
 
   if (!rows || !isRows(rows) || !isSchema(schema)) {
@@ -161,6 +166,32 @@ export async function handleDataImport(user: any, table: any, rows: any) {
 
   let finalData: any = importToRows(data, table, user)
 
+  //Set IDs of finalData to match existing row if an update is expected
+  if (identifierFields.length > 0) {
+    const allDocs = await db.allDocs(
+      getRowParams(table._id, null, {
+        include_docs: true,
+      })
+    )
+    allDocs.rows
+      .map(existingRow => existingRow.doc)
+      .forEach((doc: any) => {
+        finalData.forEach((finalItem: any) => {
+          let match = true
+          for (const field of identifierFields) {
+            if (finalItem[field] !== doc[field]) {
+              match = false
+              break
+            }
+          }
+          if (match) {
+            finalItem._id = doc._id
+            finalItem._rev = doc._rev
+          }
+        })
+      })
+  }
+
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })

packages/server/src/automations/actions.ts

@@ -74,10 +74,15 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
 // ran at all
 if (env.SELF_HOSTED) {
   const bash = require("./steps/bash")
+  const openai = require("./steps/openai")
+
   // @ts-ignore
   ACTION_IMPLS["EXECUTE_BASH"] = bash.run
   // @ts-ignore
   BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+
+  ACTION_IMPLS.OPENAI = openai.run
+  BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition
 }
 
 export async function getActionDefinitions() {

packages/server/src/automations/steps/openai.ts

@@ -0,0 +1,105 @@
+import { Configuration, OpenAIApi } from "openai"
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+} from "@budibase/types"
+import * as automationUtils from "../automationUtils"
+import environment from "../../environment"
+
+enum Model {
+  GPT_35_TURBO = "gpt-3.5-turbo",
+  // will only work with api keys that have access to the GPT4 API
+  GPT_4 = "gpt-4",
+}
+
+export const definition: AutomationStepSchema = {
+  name: "OpenAI",
+  tagline: "Send prompts to ChatGPT",
+  icon: "Algorithm",
+  description: "Interact with the OpenAI ChatGPT API.",
+  type: AutomationStepType.ACTION,
+  internal: true,
+  stepId: AutomationActionStepId.OPENAI,
+  inputs: {
+    prompt: "",
+  },
+  schema: {
+    inputs: {
+      properties: {
+        prompt: {
+          type: AutomationIOType.STRING,
+          title: "Prompt",
+        },
+        model: {
+          type: AutomationIOType.STRING,
+          title: "Model",
+          enum: Object.values(Model),
+        },
+      },
+      required: ["prompt", "model"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether the action was successful",
+        },
+        response: {
+          type: AutomationIOType.STRING,
+          description: "What was output",
+        },
+      },
+      required: ["success", "response"],
+    },
+  },
+}
+
+export async function run({ inputs, context }: AutomationStepInput) {
+  if (!environment.OPENAI_API_KEY) {
+    return {
+      success: false,
+      response:
+        "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
+    }
+  }
+
+  if (inputs.prompt == null) {
+    return {
+      success: false,
+      response: "Budibase OpenAI Automation Failed: No prompt supplied",
+    }
+  }
+
+  try {
+    const configuration = new Configuration({
+      apiKey: environment.OPENAI_API_KEY,
+    })
+
+    const openai = new OpenAIApi(configuration)
+
+    const completion = await openai.createChatCompletion({
+      model: inputs.model,
+      messages: [
+        {
+          role: "user",
+          content: inputs.prompt,
+        },
+      ],
+    })
+
+    const response = completion?.data?.choices[0]?.message?.content
+
+    return {
+      response,
+      success: true,
+    }
+  } catch (err) {
+    return {
+      success: false,
+      response: automationUtils.getError(err),
+    }
+  }
+}

packages/server/src/automations/tests/openai.spec.ts

@@ -0,0 +1,86 @@
+const setup = require("./utilities")
+import environment from "../../environment"
+import openai from "openai"
+
+jest.mock(
+  "openai",
+  jest.fn(() => ({
+    Configuration: jest.fn(),
+    OpenAIApi: jest.fn(() => ({
+      createChatCompletion: jest.fn(() => ({
+        data: {
+          choices: [
+            {
+              message: {
+                content: "This is a test",
+              },
+            },
+          ],
+        },
+      })),
+    })),
+  }))
+)
+
+const OPENAI_PROMPT = "What is the meaning of life?"
+
+describe("test the openai action", () => {
+  let config = setup.getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  beforeEach(() => {
+    environment.OPENAI_API_KEY = "abc123"
+  })
+
+  afterAll(setup.afterAll)
+
+  it("should present the correct error message when the OPENAI_API_KEY variable isn't set", async () => {
+    delete environment.OPENAI_API_KEY
+
+    let res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+    expect(res.response).toEqual(
+      "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable."
+    )
+    expect(res.success).toBeFalsy()
+  })
+
+  it("should be able to receive a response from ChatGPT given a prompt", async () => {
+    const res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+    expect(res.response).toEqual("This is a test")
+    expect(res.success).toBeTruthy()
+  })
+
+  it("should present the correct error message when a prompt is not provided", async () => {
+    const res = await setup.runStep("OPENAI", {
+      prompt: null,
+    })
+    expect(res.response).toEqual(
+      "Budibase OpenAI Automation Failed: No prompt supplied"
+    )
+    expect(res.success).toBeFalsy()
+  })
+
+  it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => {
+    openai.OpenAIApi.mockImplementation(() => ({
+      createChatCompletion: jest.fn(() => {
+        throw new Error("An error occurred while calling createChatCompletion")
+      }),
+    }))
+
+    const res = await setup.runStep("OPENAI", {
+      prompt: OPENAI_PROMPT,
+    })
+
+    expect(res.response).toEqual(
+      "Error: An error occurred while calling createChatCompletion"
+    )
+    expect(res.success).toBeFalsy()
+  })
+})

packages/server/src/environment.ts

@@ -71,6 +71,7 @@ const environment = {
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
+  OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/tsconfig.json

@@ -5,14 +5,7 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "outDir": "dist",
+    "outDir": "dist"
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/shared-core": ["../shared-core/src"],
-      "@budibase/pro": ["../pro/packages/pro/src"]
-    }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],

packages/types/src/documents/app/automation.ts

@@ -58,6 +58,7 @@ export enum AutomationActionStepId {
   QUERY_ROWS = "QUERY_ROWS",
   LOOP = "LOOP",
   COLLECT = "COLLECT",
+  OPENAI = "OPENAI",
   // these used to be lowercase step IDs, maintain for backwards compat
   discord = "discord",
   slack = "slack",

packages/worker/Dockerfile

@@ -13,7 +13,7 @@ RUN yarn global add pm2
 
 
 COPY dist/package.json .
-RUN yarn
+RUN yarn install --frozen-lockfile --production=true
 # Remove unneeded data from file system to reduce image size
 RUN apk del .gyp \
     && yarn cache clean

packages/worker/package.json

@@ -38,7 +38,7 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@budibase/backend-core": "0.0.1",
-    "@budibase/pro": "0.0.1",
+    "@budibase/pro": "develop",
     "@budibase/string-templates": "0.0.1",
     "@budibase/types": "0.0.1",
     "@koa/router": "8.0.8",

packages/worker/project.json

@@ -17,6 +17,7 @@
             "output": "."
           }
         ],
+        "external": ["graphql/*", "deasync", "mock-aws-s3", "nock"],
         "format": ["cjs"],
         "esbuildOptions": {
           "outExtension": {

packages/worker/tsconfig.json

@@ -4,13 +4,7 @@
     "composite": true,
     "declaration": true,
     "sourceMap": true,
-    "baseUrl": ".",
+    "baseUrl": "."
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/pro": ["../pro/packages/pro/src"]
-    }
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],

scripts/build.js

@@ -9,6 +9,9 @@ const path = require("path")
 const { build } = require("esbuild")
 
 const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
+const {
+  default: TsconfigPathsPlugin,
+} = require("@esbuild-plugins/tsconfig-paths")
 
 var argv = require("minimist")(process.argv.slice(2))
 
@@ -23,6 +26,7 @@ function runBuild(entry, outfile) {
     sourcemap: isDev,
     tsconfig,
     plugins: [
+      TsconfigPathsPlugin({ tsconfig }),
       NodeResolve({
         extensions: [".ts", ".js"],
         onResolved: resolved => {

yarn.lock

@@ -1728,6 +1728,47 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.6.19-alpha.4":
+  version "2.6.19-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.6.19-alpha.4.tgz#525dd7ba6c5db4404cf00d1165f79d34a1093826"
+  integrity sha512-1pfOr+J9xYawedVmvqpQ4b/8C2SQP4cKhFmSz5uErM2SCgbRj+JuzOUTPNX0vzAXPvat/kEegt79xThummDvhA==
+  dependencies:
+    "@budibase/nano" "10.1.2"
+    "@budibase/pouchdb-replication-stream" "1.2.10"
+    "@budibase/types" "2.6.19-alpha.4"
+    "@shopify/jest-koa-mocks" "5.0.1"
+    "@techpass/passport-openidconnect" "0.3.2"
+    aws-cloudfront-sign "2.2.0"
+    aws-sdk "2.1030.0"
+    bcrypt "5.0.1"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dotenv "16.0.1"
+    emitter-listener "1.1.2"
+    ioredis "4.28.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.0"
+    koa-passport "4.1.4"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    lodash.isarguments "3.1.0"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-jwt "4.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "1.3.0"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "7.3.7"
+    tar-fs "2.1.1"
+    uuid "8.3.2"
+
 "@budibase/bbui@^0.9.139":
   version "0.9.190"
   resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-0.9.190.tgz#e1ec400ac90f556bfbc80fc23a04506f1585ea81"
@@ -1828,6 +1869,32 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@develop":
+  version "2.6.19-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.19-alpha.4.tgz#5d4c885ac9ac4ccfb2f8896961aca903c1178750"
+  integrity sha512-iu2QzV8Z77c00muBSK+NVsZdug3lLD0lR+vcKancGEz1PPE4yNIH7g8jB6i8h9agArbx9S2ICeHQqGb6nQaGHQ==
+  dependencies:
+    "@budibase/backend-core" "2.6.19-alpha.4"
+    "@budibase/shared-core" "2.6.19-alpha.4"
+    "@budibase/string-templates" "2.6.19-alpha.4"
+    "@budibase/types" "2.6.19-alpha.4"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    joi "17.6.0"
+    jsonwebtoken "8.5.1"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "^2.6.1"
+    scim-patch "^0.7.0"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.6.19-alpha.4":
+  version "2.6.19-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/shared-core/-/shared-core-2.6.19-alpha.4.tgz#dd22dd0a18ee4d6739b629f461e5caec90706282"
+  integrity sha512-ac6iWSsgz70OYbdA+QHPLpTnRbIZ4OecVc6Y7gnEZ78hZ4S5da8a+73jswuy0/t4YsiT/gjukjzjoihg3NemVg==
+  dependencies:
+    "@budibase/types" "2.6.19-alpha.4"
+
 "@budibase/standard-components@^0.9.139":
   version "0.9.139"
   resolved "https://registry.yarnpkg.com/@budibase/standard-components/-/standard-components-0.9.139.tgz#cf8e2b759ae863e469e50272b3ca87f2827e66e3"
@@ -1846,6 +1913,25 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
+"@budibase/string-templates@2.6.19-alpha.4":
+  version "2.6.19-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-2.6.19-alpha.4.tgz#92ebd69a6841174b8af91f338c4754ca7c402707"
+  integrity sha512-KsH3NlQcJibRj98Q8zQ3KQHhfSIWPQfvR80MmBTIe05llEZGox4re4pQQUnlMafaUEyNNtIqVnbTJ1XP0LmFng==
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.11.8"
+    dayjs "^1.10.4"
+    handlebars "^4.7.6"
+    handlebars-utils "^1.0.6"
+    lodash "^4.17.20"
+    vm2 "^3.9.15"
+
+"@budibase/types@2.6.19-alpha.4":
+  version "2.6.19-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.6.19-alpha.4.tgz#bcf81699329d3f8509e4b0a489211f35b6cfa7ce"
+  integrity sha512-qFsXHZTSigcfCv02aTZGsf17vBT/MC+zK9ky7WZVX4h0sJiE0li4A66/tMaSDz3/vQ7ToPRhJK/p+LOWA/oceg==
+  dependencies:
+    scim-patch "^0.7.0"
+
 "@bull-board/api@3.7.0":
   version "3.7.0"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"
@@ -2047,6 +2133,15 @@
     escape-string-regexp "^4.0.0"
     resolve "^1.19.0"
 
+"@esbuild-plugins/tsconfig-paths@^0.1.2":
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/@esbuild-plugins/tsconfig-paths/-/tsconfig-paths-0.1.2.tgz#1955de0a124ecf4364717a2fadbfbea876955232"
+  integrity sha512-TusFR26Y+Ze+Zm+NdfqZTSG4XyrXKxIaAfYCL3jASEI/gHjSdoCujATjzNWaaXs6Sk6Bv2D7NLr4Jdz1gysy/Q==
+  dependencies:
+    debug "^4.3.1"
+    find-up "^5.0.0"
+    strip-json-comments "^3.1.1"
+
 "@esbuild/android-arm64@0.16.17":
   version "0.16.17"
   resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.16.17.tgz#cf91e86df127aa3d141744edafcba0abdc577d23"
@@ -3448,11 +3543,6 @@
   resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
   integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
 
-"@jridgewell/sourcemap-codec@^1.4.13":
-  version "1.4.15"
-  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
-  integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==
-
 "@jridgewell/trace-mapping@0.3.9":
   version "0.3.9"
   resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
@@ -4233,7 +4323,7 @@
   dependencies:
     slash "^3.0.0"
 
-"@rollup/plugin-commonjs@16.0.0", "@rollup/plugin-commonjs@^16.0.0":
+"@rollup/plugin-commonjs@^16.0.0":
   version "16.0.0"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-commonjs/-/plugin-commonjs-16.0.0.tgz#169004d56cd0f0a1d0f35915d31a036b0efe281f"
   integrity sha512-LuNyypCP3msCGVQJ7ki8PqYdpjfEkE/xtFa5DqlF+7IBD0JsfMZ87C58heSwIMint58sAUZbt3ITqOmdQv/dXw==
@@ -4316,22 +4406,6 @@
     "@rollup/pluginutils" "^3.1.0"
     magic-string "^0.25.7"
 
-"@rollup/plugin-replace@^5.0.2":
-  version "5.0.2"
-  resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.2.tgz#45f53501b16311feded2485e98419acb8448c61d"
-  integrity sha512-M9YXNekv/C/iHHK+cvORzfRYfPbq0RDD8r0G+bMiTXjNGKulPnCT9O3Ss46WfhI6ZOCgApOP7xAdmCQJ+U2LAA==
-  dependencies:
-    "@rollup/pluginutils" "^5.0.1"
-    magic-string "^0.27.0"
-
-"@rollup/plugin-typescript@8.3.0":
-  version "8.3.0"
-  resolved "https://registry.yarnpkg.com/@rollup/plugin-typescript/-/plugin-typescript-8.3.0.tgz#bc1077fa5897b980fc27e376c4e377882c63e68b"
-  integrity sha512-I5FpSvLbtAdwJ+naznv+B4sjXZUcIvLLceYpITAn7wAP8W0wqc5noLdGIp9HGVntNhRWXctwPYrSSFQxtl0FPA==
-  dependencies:
-    "@rollup/pluginutils" "^3.1.0"
-    resolve "^1.17.0"
-
 "@rollup/pluginutils@^3.0.8", "@rollup/pluginutils@^3.1.0":
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/@rollup/pluginutils/-/pluginutils-3.1.0.tgz#706b4524ee6dc8b103b3c995533e5ad680c02b9b"
@@ -12085,6 +12159,14 @@ find-up@^4.0.0, find-up@^4.1.0:
     locate-path "^5.0.0"
     path-exists "^4.0.0"
 
+find-up@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc"
+  integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==
+  dependencies:
+    locate-path "^6.0.0"
+    path-exists "^4.0.0"
+
 findit2@^2.2.3:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/findit2/-/findit2-2.2.3.tgz#58a466697df8a6205cdfdbf395536b8bd777a5f6"
@@ -12351,7 +12433,7 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
-fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.1, fsevents@~2.3.2:
+fsevents@^2.1.2, fsevents@^2.3.2, fsevents@~2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
@@ -17012,6 +17094,13 @@ locate-path@^5.0.0:
   dependencies:
     p-locate "^4.1.0"
 
+locate-path@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286"
+  integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==
+  dependencies:
+    p-locate "^5.0.0"
+
 lodash-es@^4.17.11:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
@@ -17392,13 +17481,6 @@ magic-string@^0.26.2:
   dependencies:
     sourcemap-codec "^1.4.8"
 
-magic-string@^0.27.0:
-  version "0.27.0"
-  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.27.0.tgz#e4a3413b4bab6d98d2becffd48b4a257effdbbf3"
-  integrity sha512-8UnnX2PeRAPZuN12svgR9j7M1uWMovg/CEnIwIG0LFkXSJJe4PdfUGiTGl8V9bsBHFUtfVINcSyYxd7q+kx9fA==
-  dependencies:
-    "@jridgewell/sourcemap-codec" "^1.4.13"
-
 make-dir@3.1.0, make-dir@^3.0.0, make-dir@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@@ -19149,6 +19231,14 @@ open@^8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
+openai@^3.2.1:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-3.2.1.tgz#1fa35bdf979cbde8453b43f2dd3a7d401ee40866"
+  integrity sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==
+  dependencies:
+    axios "^0.26.0"
+    form-data "^4.0.0"
+
 openapi-response-validator@^9.2.0:
   version "9.3.1"
   resolved "https://registry.yarnpkg.com/openapi-response-validator/-/openapi-response-validator-9.3.1.tgz#54284d8be608ef53283cbe7448accce8106b1c56"
@@ -19371,7 +19461,7 @@ p-limit@^2.0.0, p-limit@^2.2.0:
   dependencies:
     p-try "^2.0.0"
 
-p-limit@^3.1.0:
+p-limit@^3.0.2, p-limit@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b"
   integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==
@@ -19406,6 +19496,13 @@ p-locate@^4.1.0:
   dependencies:
     p-limit "^2.2.0"
 
+p-locate@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834"
+  integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==
+  dependencies:
+    p-limit "^3.0.2"
+
 p-map-series@2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/p-map-series/-/p-map-series-2.1.0.tgz#7560d4c452d9da0c07e692fdbfe6e2c81a2a91f2"
@@ -22210,13 +22307,6 @@ rollup-pluginutils@^2.3.1, rollup-pluginutils@^2.5.0, rollup-pluginutils@^2.6.0,
   dependencies:
     estree-walker "^0.6.1"
 
-rollup@2.45.2:
-  version "2.45.2"
-  resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.45.2.tgz#8fb85917c9f35605720e92328f3ccbfba6f78b48"
-  integrity sha512-kRRU7wXzFHUzBIv0GfoFFIN3m9oteY4uAsKllIpQDId5cfnkWF2J130l+27dzDju0E6MScKiV0ZM5Bw8m4blYQ==
-  optionalDependencies:
-    fsevents "~2.3.1"
-
 rollup@^2.36.2, rollup@^2.44.0, rollup@^2.45.2, rollup@^2.79.1:
   version "2.79.1"
   resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.79.1.tgz#bedee8faef7c9f93a2647ac0108748f497f081c7"
@@ -24222,7 +24312,7 @@ timed-out@^4.0.1:
   resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f"
   integrity sha512-G7r3AhovYtr5YKOWQkta8RKAPb+J9IsO4uVmzjl8AZwfhs8UcUwTiD6gcJYSgOtzyjvQKrKYn41syHbUWMkafA==
 
-timekeeper@2.2.0, timekeeper@^2.2.0:
+timekeeper@2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368"
   integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==