Merge remote-tracking branch 'origin/develop' into feat/relationship-configuration

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.12-alpha.23",
+  "version": "2.10.12-alpha.26",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -21,7 +21,7 @@
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
-    "svelte": "^3.38.2",
+    "svelte": "3.49.0",
     "typescript": "5.2.2",
     "@babel/core": "^7.22.5",
     "@babel/eslint-parser": "^7.22.5",

packages/backend-core/package.json

@@ -33,17 +33,14 @@
     "bull": "4.10.1",
     "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
-    "emitter-listener": "1.1.2",
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
-    "lodash.isarguments": "3.1.0",
     "node-fetch": "2.6.7",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "passport-oauth2-refresh": "^2.1.0",
     "pino": "8.11.0",
@@ -59,14 +56,13 @@
     "uuid": "8.3.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.6.2",
     "@shopify/jest-koa-mocks": "5.1.1",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
+    "@types/cookies": "0.7.8",
     "@types/jest": "29.5.3",
-    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
@@ -80,13 +76,9 @@
     "jest": "29.6.2",
     "jest-environment-node": "29.6.2",
     "jest-serial-runner": "1.2.1",
-    "koa": "2.13.4",
-    "nodemon": "2.0.16",
     "pino-pretty": "10.0.0",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
     "typescript": "5.2.2"
   },
   "nx": {

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,7 @@ import {
   Event,
   TenantResolutionStrategy,
 } from "@budibase/types"
-import { SetOption } from "cookies"
+import type { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/bbui/package.json

@@ -20,14 +20,12 @@
     "@rollup/plugin-commonjs": "^16.0.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
-    "cross-env": "^7.0.2",
-    "nollup": "^0.14.1",
     "postcss": "^8.2.9",
     "rollup": "^2.45.2",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "svelte": "^3.38.2"
+    "svelte": "3.49.0"
   },
   "keywords": [
     "svelte"

packages/builder/package.json

@@ -65,7 +65,6 @@
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
     "@sentry/browser": "5.19.1",
-    "@spectrum-css/accordion": "^3.0.24",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
@@ -75,18 +74,17 @@
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",
+    "sanitize-html": "^2.7.0",
     "shortid": "2.2.15",
     "svelte-dnd-action": "^0.9.8",
     "svelte-loading-spinners": "^0.1.1",
     "svelte-portal": "1.0.0",
-    "uuid": "8.3.1",
     "yup": "0.29.2"
   },
   "devDependencies": {
     "@babel/core": "^7.12.14",
     "@babel/plugin-transform-runtime": "^7.13.10",
     "@babel/preset-env": "^7.13.12",
-    "@babel/runtime": "^7.13.10",
     "@rollup/plugin-replace": "^2.4.2",
     "@roxi/routify": "2.18.5",
     "@sveltejs/vite-plugin-svelte": "1.0.1",
@@ -96,19 +94,10 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.6.2",
     "jsdom": "^21.1.1",
-    "mochawesome": "^7.1.3",
-    "mochawesome-merge": "^4.2.1",
-    "mochawesome-report-generator": "^6.2.0",
     "ncp": "^2.0.0",
-    "rimraf": "^3.0.2",
     "rollup": "^2.44.0",
-    "rollup-plugin-copy": "^3.4.0",
-    "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
-    "typescript": "5.2.2",
     "vite": "^3.0.8",
     "vite-plugin-static-copy": "^0.16.0",
     "vitest": "^0.29.2"

packages/builder/src/stores/backend/tables.js

@@ -1,7 +1,7 @@
 import { get, writable, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "api"
-import { SWITCHABLE_TYPES } from "constants/backend"
+import { SWITCHABLE_TYPES, FIELDS } from "constants/backend"
 
 export function createTablesStore() {
   const store = writable({
@@ -21,6 +21,23 @@ export function createTablesStore() {
     }))
   }
 
+  const singleFetch = async tableId => {
+    const table = await API.getTable(tableId)
+    store.update(state => {
+      const list = []
+      // update the list, keep order accurate
+      for (let tbl of state.list) {
+        if (table._id === tbl._id) {
+          list.push(table)
+        } else {
+          list.push(tbl)
+        }
+      }
+      state.list = list
+      return state
+    })
+  }
+
   const select = tableId => {
     store.update(state => ({
       ...state,
@@ -63,6 +80,20 @@ export function createTablesStore() {
     const savedTable = await API.saveTable(updatedTable)
     replaceTable(savedTable._id, savedTable)
     select(savedTable._id)
+    // make sure tables up to date (related)
+    let tableIdsToFetch = []
+    for (let column of Object.values(updatedTable?.schema || {})) {
+      if (column.type === FIELDS.LINK.type) {
+        tableIdsToFetch.push(column.tableId)
+      }
+    }
+    tableIdsToFetch = [...new Set(tableIdsToFetch)]
+    // too many tables to fetch, just get all
+    if (tableIdsToFetch.length > 3) {
+      await fetch()
+    } else {
+      await Promise.all(tableIdsToFetch.map(id => singleFetch(id)))
+    }
     return savedTable
   }
 

packages/builder/tsconfig.json

@@ -5,8 +5,5 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": "."
-  },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"]
   }
 }

packages/cli/package.json

@@ -32,7 +32,6 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
-    "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
     "commander": "7.1.0",
@@ -41,7 +40,6 @@
     "download": "8.0.0",
     "find-free-port": "^2.0.0",
     "inquirer": "8.0.0",
-    "joi": "17.6.0",
     "lookpath": "1.1.0",
     "node-fetch": "2.6.7",
     "pkg": "5.8.0",
@@ -53,13 +51,9 @@
     "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@swc/core": "1.3.71",
-    "@swc/jest": "0.2.27",
     "@types/jest": "29.5.3",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "^6.4.0",
-    "copyfiles": "^2.4.1",
-    "eslint": "^7.20.0",
     "renamer": "^4.0.0",
     "ts-node": "^10.9.1",
     "typescript": "5.2.2"

packages/frontend-core/src/api/tables.js

@@ -82,8 +82,9 @@ export const buildTableEndpoints = API => ({
       },
     })
   },
+
   /**
-   * Gets a list o tables.
+   * Gets a list of tables.
    */
   getTables: async () => {
     return await API.get({
@@ -91,6 +92,15 @@ export const buildTableEndpoints = API => ({
     })
   },
 
+  /**
+   * Get a single table based on table ID.
+   */
+  getTable: async tableId => {
+    return await API.get({
+      url: `/api/tables/${tableId}`,
+    })
+  },
+
   /**
    * Saves a table.
    * @param table the table to save

packages/sdk/package.json

@@ -10,14 +10,10 @@
     "generate": "cd scripts && bash generate-sdk.sh",
     "build:sdk": "yarn run generate && rollup -c"
   },
-  "dependencies": {
-    "superagent": "^5.3.0"
-  },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^18.0.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "rollup": "^2.44.0",
-    "rollup-plugin-polyfill-node": "^0.8.0",
     "rollup-plugin-terser": "^7.0.2"
   }
 }

packages/server/package.json

@@ -57,7 +57,6 @@
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "5.0.2",
     "@koa/router": "8.0.8",
-    "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
     "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
@@ -66,18 +65,14 @@
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
-    "chmodr": "1.2.0",
     "chokidar": "3.5.3",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dd-trace": "3.13.2",
     "dotenv": "8.2.0",
-    "download": "8.0.0",
-    "elastic-apm-node": "3.38.0",
     "fix-path": "3.0.0",
     "form-data": "4.0.0",
-    "fs-extra": "8.1.0",
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
@@ -90,10 +85,7 @@
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
-    "koa-connect": "2.1.0",
     "koa-send": "5.0.0",
-    "koa-session": "5.12.0",
-    "koa-static": "5.0.0",
     "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
@@ -108,7 +100,6 @@
     "pg": "8.10.0",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
-    "pouchdb-adapter-memory": "7.2.2",
     "pouchdb-all-dbs": "1.0.2",
     "pouchdb-find": "7.2.2",
     "pouchdb-replication-stream": "1.2.9",
@@ -116,8 +107,7 @@
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.6.1",
-    "svelte": "3.49.0",
+    "svelte": "^3.49.0",
-    "swagger-parser": "10.0.3",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
@@ -130,13 +120,9 @@
   "devDependencies": {
     "@babel/core": "7.17.4",
     "@babel/preset-env": "7.16.11",
-    "@budibase/standard-components": "^0.9.139",
-    "@jest/test-sequencer": "29.6.2",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@trendyol/jest-testcontainers": "2.1.1",
-    "@types/apidoc": "0.50.0",
-    "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
     "@types/jest": "29.5.3",
@@ -148,17 +134,12 @@
     "@types/node-fetch": "2.6.4",
     "@types/oracledb": "5.2.2",
     "@types/pg": "8.6.6",
-    "@types/pouchdb": "6.4.0",
-    "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/tar": "6.1.5",
     "apidoc": "0.50.4",
-    "babel-jest": "29.6.2",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
-    "eslint": "6.8.0",
-    "is-wsl": "2.2.0",
     "jest": "29.6.2",
     "jest-openapi": "0.14.2",
     "jest-runner": "29.6.2",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -818,7 +818,10 @@ export class ExternalRequest {
     // can't really use response right now
     const response = await getDatasourceAndQuery(json)
     // handle many to many relationships now if we know the ID (could be auto increment)
-    if (operation !== Operation.READ && processed.manyRelationships) {
+    if (
+      operation !== Operation.READ &&
+      processed.manyRelationships?.length > 0
+    ) {
       await this.handleManyRelationships(
         table._id || "",
         response[0],

packages/server/src/api/controllers/table/index.ts

@@ -14,10 +14,12 @@ import {
   Table,
   TableResponse,
   UserCtx,
+  Datasource,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
+import { cloneDeep } from "lodash"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -35,16 +37,16 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
 export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
   const internal = await sdk.tables.getAllInternalTables()
 
-  const externalTables = await sdk.datasources.getExternalDatasources()
+  const datasources = await sdk.datasources.getExternalDatasources()
 
-  const external = externalTables.flatMap(table => {
+  const external = datasources.flatMap(datasource => {
-    let entities = table.entities
+    let entities = datasource.entities
     if (entities) {
       return Object.values(entities).map<Table>((entity: Table) => ({
         ...entity,
         type: "external",
-        sourceId: table._id,
+        sourceId: datasource._id,
-        sql: isSQL(table),
+        sql: isSQL(datasource),
       }))
     } else {
       return []
@@ -80,7 +82,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
+  builderSocket?.emitTableUpdate(ctx, cloneDeep(savedTable))
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/integrations/utils.ts

@@ -1,4 +1,4 @@
-import { SqlQuery, Table, SearchFilters } from "@budibase/types"
+import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import {
   FieldTypes,
@@ -184,7 +184,9 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery {
   }
 }
 
-export const isSQL = helpers.isSQL
+export function isSQL(datasource: Datasource) {
+  return helpers.isSQL(datasource)
+}
 
 export function isIsoDateString(str: string) {
   const trimmedValue = str.trim()

packages/shared-core/package.json

@@ -17,7 +17,6 @@
     "@budibase/types": "0.0.0"
   },
   "devDependencies": {
-    "concurrently": "^7.6.0",
     "rimraf": "3.0.2",
     "typescript": "5.2.2"
   },

packages/string-templates/package.json

@@ -28,7 +28,6 @@
     "@budibase/handlebars-helpers": "^0.11.9",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
-    "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",
     "vm2": "^3.9.15"
   },
@@ -37,7 +36,6 @@
     "@rollup/plugin-json": "^4.1.0",
     "doctrine": "^3.0.0",
     "jest": "29.6.2",
-    "jest-environment-node": "29.6.2",
     "marked": "^4.0.10",
     "rollup": "^2.36.2",
     "rollup-plugin-inject-process-env": "^1.3.1",

packages/types/package.json

@@ -16,13 +16,10 @@
   "jest": {},
   "devDependencies": {
     "@budibase/nano": "10.1.2",
-    "@types/json5": "2.2.0",
     "@types/koa": "2.13.4",
     "@types/node": "18.17.0",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",
-    "concurrently": "^7.6.0",
-    "koa-body": "4.2.0",
     "rimraf": "3.0.2",
     "typescript": "5.2.2"
   },

packages/types/src/sdk/db.ts

@@ -1,7 +1,6 @@
 import Nano from "@budibase/nano"
 import { AllDocsResponse, AnyDocument, Document } from "../"
 import { Writable } from "stream"
-import PouchDB from "pouchdb"
 
 export enum SearchIndex {
   ROWS = "rows",

packages/worker/package.json

@@ -67,7 +67,6 @@
     "node-fetch": "2.6.7",
     "nodemailer": "6.7.2",
     "passport-google-oauth": "2.0.0",
-    "passport-jwt": "4.0.0",
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
@@ -84,21 +83,15 @@
     "@types/lodash": "^4.14.191",
     "@types/node": "18.17.0",
     "@types/node-fetch": "2.6.4",
-    "@types/pouchdb": "6.4.0",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.12",
     "@types/uuid": "8.3.4",
-    "copyfiles": "2.4.1",
-    "eslint": "6.8.0",
     "jest": "29.6.2",
     "lodash": "4.17.21",
     "nodemon": "2.0.15",
-    "pouchdb-adapter-memory": "7.2.2",
     "rimraf": "3.0.2",
     "supertest": "6.2.2",
     "timekeeper": "2.2.0",
-    "ts-node": "10.8.1",
-    "tsconfig-paths": "4.0.0",
     "typescript": "5.2.2",
     "update-dotenv": "1.1.1"
   },

packages/worker/src/api/routes/global/tests/roles.spec.ts

@@ -1,6 +1,5 @@
 import { structures, TestConfiguration } from "../../../../tests"
 import { context, db, permissions, roles } from "@budibase/backend-core"
-import { Mock } from "jest-mock"
 import { Database } from "@budibase/types"
 
 jest.mock("@budibase/backend-core", () => {
@@ -47,7 +46,7 @@ describe("/api/global/roles", () => {
   beforeEach(async () => {
     appId = db.generateAppID(config.tenantId)
     appDb = db.getDB(appId)
-    const mockAppDB = context.getAppDB as Mock
+    const mockAppDB = context.getAppDB as jest.Mock
     mockAppDB.mockReturnValue(appDb)
 
     await addAppMetadata()

packages/worker/tsconfig.json

@@ -4,10 +4,6 @@
     "composite": true,
     "baseUrl": "."
   },
-  "ts-node": {
-    "require": ["tsconfig-paths/register"],
-    "swc": true
-  },
   "include": ["src/**/*", "__mocks__/**/*"],
   "exclude": ["node_modules", "dist"]
 }

scripts/syncProPackage.js

@@ -33,7 +33,7 @@ Object.keys(data).forEach(workspace => {
 
   let hasChanges = false
 
-  if (packageJson.dependencies["@budibase/pro"]) {
+  if (packageJson.dependencies && packageJson.dependencies["@budibase/pro"]) {
     packageJson.dependencies["@budibase/pro"] = version
     hasChanges = true
   }