Merge branches 'master' and 'fix-automation-loop-test-output-2' of github.com:budibase/budibase into fix-automation-loop-test-output-2

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.7",
+  "version": "3.4.10",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/utils/utils.ts

@@ -247,3 +247,7 @@ export function hasCircularStructure(json: any) {
   }
   return false
 }
+
+export function urlHasProtocol(url: string): boolean {
+  return !!url.match(/^.+:\/\/.+$/)
+}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/dndStore.js

@@ -1,12 +1,8 @@
 import { writable, get } from "svelte/store"
 import { findComponentParent, findComponentPath } from "@/helpers/components"
 import { selectedScreen, componentStore } from "@/stores/builder"
-
-export const DropPosition = {
-  ABOVE: "above",
-  BELOW: "below",
-  INSIDE: "inside",
-}
+import { DropPosition } from "@budibase/types"
+export { DropPosition } from "@budibase/types"
 
 const initialState = {
   source: null,

packages/builder/src/stores/builder/index.js

@@ -1,20 +1,20 @@
-import { layoutStore } from "./layouts.js"
-import { appStore } from "./app.js"
+import { layoutStore } from "./layouts"
+import { appStore } from "./app"
 import { componentStore, selectedComponent } from "./components"
-import { navigationStore } from "./navigation.js"
-import { themeStore } from "./theme.js"
+import { navigationStore } from "./navigation"
+import { themeStore } from "./theme"
 import { screenStore, selectedScreen, sortedScreens } from "./screens"
-import { builderStore } from "./builder.js"
-import { hoverStore } from "./hover.js"
-import { previewStore } from "./preview.js"
+import { builderStore } from "./builder"
+import { hoverStore } from "./hover"
+import { previewStore } from "./preview"
 import {
   automationStore,
   selectedAutomation,
   automationHistoryStore,
-} from "./automations.js"
-import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
-import { deploymentStore } from "./deployments.js"
-import { contextMenuStore } from "./contextMenu.js"
+} from "./automations"
+import { userStore, userSelectedResourceMap, isOnlyUser } from "./users"
+import { deploymentStore } from "./deployments"
+import { contextMenuStore } from "./contextMenu"
 import { snippets } from "./snippets"
 import {
   screenComponentsList,

packages/client/src/components/Block.svelte

@@ -1,6 +1,6 @@
 <script>
   import { getContext, onDestroy, onMount, setContext } from "svelte"
-  import { builderStore } from "@/stores/builder.js"
+  import { builderStore } from "@/stores/builder"
   import { blockStore } from "@/stores/blocks"
 
   const component = getContext("component")

packages/client/src/components/BlockComponent.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, onDestroy } from "svelte"
   import { generate } from "shortid"
-  import { builderStore } from "../stores/builder.js"
+  import { builderStore } from "../stores/builder"
   import Component from "@/components/Component.svelte"
 
   export let type

packages/client/src/components/app/forms/S3Upload.svelte

@@ -2,7 +2,7 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
-  import { builderStore } from "@/stores/builder.js"
+  import { builderStore } from "@/stores/builder"
   import { processStringSync } from "@budibase/string-templates"
 
   export let datasourceId

packages/client/src/index.ts

@@ -19,7 +19,6 @@ import type { ActionTypes } from "@/constants"
 import { Readable } from "svelte/store"
 import {
   Screen,
-  Layout,
   Theme,
   AppCustomTheme,
   PreviewDevice,
@@ -48,7 +47,6 @@ declare global {
     // Data from builder
     "##BUDIBASE_APP_ID##"?: string
     "##BUDIBASE_IN_BUILDER##"?: true
-    "##BUDIBASE_PREVIEW_LAYOUT##"?: Layout
     "##BUDIBASE_PREVIEW_SCREEN##"?: Screen
     "##BUDIBASE_SELECTED_COMPONENT_ID##"?: string
     "##BUDIBASE_PREVIEW_ID##"?: number
@@ -59,13 +57,8 @@ declare global {
     "##BUDIBASE_PREVIEW_NAVIGATION##"?: AppNavigation
     "##BUDIBASE_HIDDEN_COMPONENT_IDS##"?: string[]
     "##BUDIBASE_USED_PLUGINS##"?: Plugin[]
-    "##BUDIBASE_LOCATION##"?: {
-      protocol: string
-      hostname: string
-      port: string
-    }
     "##BUDIBASE_SNIPPETS##"?: Snippet[]
-    "##BUDIBASE_COMPONENT_ERRORS##"?: Record<string, UIComponentError>[]
+    "##BUDIBASE_COMPONENT_ERRORS##"?: Record<string, UIComponentError[]>
     "##BUDIBASE_CUSTOM_COMPONENTS##"?: CustomComponent[]
 
     // Other flags
@@ -115,7 +108,6 @@ const loadBudibase = async () => {
   builderStore.set({
     ...get(builderStore),
     inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
@@ -125,7 +117,6 @@ const loadBudibase = async () => {
     navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
     hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
     usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
-    location: window["##BUDIBASE_LOCATION##"],
     snippets: window["##BUDIBASE_SNIPPETS##"],
     componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
   })

packages/client/src/stores/builder.ts

@@ -2,9 +2,39 @@ import { writable, get } from "svelte/store"
 import { API } from "@/api"
 import { devToolsStore } from "./devTools.js"
 import { eventStore } from "./events.js"
+import {
+  ComponentDefinition,
+  DropPosition,
+  PingSource,
+  PreviewDevice,
+  Screen,
+  Theme,
+  AppCustomTheme,
+  AppNavigation,
+  Plugin,
+  Snippet,
+  UIComponentError,
+} from "@budibase/types"
+
+interface BuilderStore {
+  inBuilder: boolean
+  screen?: Screen | null
+  selectedComponentId?: string | null
+  editMode: boolean
+  previewId?: number | null
+  theme?: Theme | null
+  customTheme?: AppCustomTheme | null
+  previewDevice?: PreviewDevice
+  navigation?: AppNavigation | null
+  hiddenComponentIds?: string[]
+  usedPlugins?: Plugin[] | null
+  metadata: { componentId: string; step: number } | null
+  snippets?: Snippet[] | null
+  componentErrors?: Record<string, UIComponentError[]>
+}
 
 const createBuilderStore = () => {
-  const initialState = {
+  const initialState: BuilderStore = {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
@@ -16,17 +46,13 @@ const createBuilderStore = () => {
     navigation: null,
     hiddenComponentIds: [],
     usedPlugins: null,
-    eventResolvers: {},
     metadata: null,
     snippets: null,
     componentErrors: {},
-
-    // Legacy - allow the builder to specify a layout
-    layout: null,
   }
   const store = writable(initialState)
   const actions = {
-    selectComponent: id => {
+    selectComponent: (id: string) => {
       if (id === get(store).selectedComponentId) {
         return
       }
@@ -38,46 +64,59 @@ const createBuilderStore = () => {
       devToolsStore.actions.setAllowSelection(false)
       eventStore.actions.dispatchEvent("select-component", { id })
     },
-    updateProp: (prop, value) => {
+    updateProp: (prop: string, value: any) => {
       eventStore.actions.dispatchEvent("update-prop", { prop, value })
     },
-    updateStyles: async (styles, id) => {
+    updateStyles: async (styles: Record<string, any>, id: string) => {
       await eventStore.actions.dispatchEvent("update-styles", {
         styles,
         id,
       })
     },
-    keyDown: (key, ctrlKey) => {
+    keyDown: (key: string, ctrlKey: boolean) => {
       eventStore.actions.dispatchEvent("key-down", { key, ctrlKey })
     },
-    duplicateComponent: (id, mode = "below", selectComponent = true) => {
+    duplicateComponent: (
+      id: string,
+      mode = DropPosition.BELOW,
+      selectComponent = true
+    ) => {
       eventStore.actions.dispatchEvent("duplicate-component", {
         id,
         mode,
         selectComponent,
       })
     },
-    deleteComponent: id => {
+    deleteComponent: (id: string) => {
       eventStore.actions.dispatchEvent("delete-component", { id })
     },
     notifyLoaded: () => {
       eventStore.actions.dispatchEvent("preview-loaded")
     },
-    analyticsPing: async ({ embedded }) => {
+    analyticsPing: async ({ embedded }: { embedded: boolean }) => {
       try {
-        await API.analyticsPing({ source: "app", embedded })
+        await API.analyticsPing({ source: PingSource.APP, embedded })
       } catch (error) {
         // Do nothing
       }
     },
-    moveComponent: async (componentId, destinationComponentId, mode) => {
+    moveComponent: async (
+      componentId: string,
+      destinationComponentId: string,
+      mode: DropPosition
+    ) => {
       await eventStore.actions.dispatchEvent("move-component", {
         componentId,
         destinationComponentId,
         mode,
       })
     },
-    dropNewComponent: (component, parent, index, props) => {
+    dropNewComponent: (
+      component: string,
+      parent: string,
+      index: number,
+      props: Record<string, any>
+    ) => {
       eventStore.actions.dispatchEvent("drop-new-component", {
         component,
         parent,
@@ -85,7 +124,7 @@ const createBuilderStore = () => {
         props,
       })
     },
-    setEditMode: enabled => {
+    setEditMode: (enabled: boolean) => {
       if (enabled === get(store).editMode) {
         return
       }
@@ -94,18 +133,18 @@ const createBuilderStore = () => {
     requestAddComponent: () => {
       eventStore.actions.dispatchEvent("request-add-component")
     },
-    highlightSetting: setting => {
+    highlightSetting: (setting: string) => {
       eventStore.actions.dispatchEvent("highlight-setting", { setting })
     },
-    ejectBlock: (id, definition) => {
+    ejectBlock: (id: string, definition: ComponentDefinition) => {
       eventStore.actions.dispatchEvent("eject-block", { id, definition })
     },
-    updateUsedPlugin: (name, hash) => {
+    updateUsedPlugin: (name: string, hash: string) => {
       // Check if we used this plugin
       const used = get(store)?.usedPlugins?.find(x => x.name === name)
       if (used) {
         store.update(state => {
-          state.usedPlugins = state.usedPlugins.filter(x => x.name !== name)
+          state.usedPlugins = state.usedPlugins!.filter(x => x.name !== name)
           state.usedPlugins.push({
             ...used,
             hash,
@@ -117,13 +156,13 @@ const createBuilderStore = () => {
       // Notify the builder so we can reload component definitions
       eventStore.actions.dispatchEvent("reload-plugin")
     },
-    addParentComponent: (componentId, parentType) => {
+    addParentComponent: (componentId: string, parentType: string) => {
       eventStore.actions.dispatchEvent("add-parent-component", {
         componentId,
         parentType,
       })
     },
-    setMetadata: metadata => {
+    setMetadata: (metadata: { componentId: string; step: number }) => {
       store.update(state => ({
         ...state,
         metadata,
@@ -132,7 +171,7 @@ const createBuilderStore = () => {
   }
   return {
     ...store,
-    set: state => store.set({ ...initialState, ...state }),
+    set: (state: BuilderStore) => store.set({ ...initialState, ...state }),
     actions,
   }
 }

packages/client/src/stores/derived/devToolsEnabled.js

@@ -1,6 +1,6 @@
 import { derived } from "svelte/store"
 import { appStore } from "../app.js"
-import { builderStore } from "../builder.js"
+import { builderStore } from "../builder"
 
 export const devToolsEnabled = derived(
   [appStore, builderStore],

packages/client/src/stores/derived/snippets.js

@@ -1,5 +1,5 @@
 import { appStore } from "../app.js"
-import { builderStore } from "../builder.js"
+import { builderStore } from "../builder"
 import { derivedMemo } from "@budibase/frontend-core"
 
 export const snippets = derivedMemo(

packages/client/src/stores/screens.js

@@ -36,11 +36,6 @@ const createScreenStore = () => {
         activeScreen = Helpers.cloneDeep($builderStore.screen)
         screens = [activeScreen]
 
-        // Legacy - allow the builder to specify a layout
-        if ($builderStore.layout) {
-          activeLayout = $builderStore.layout
-        }
-
         // Attach meta
         const errors = $builderStore.componentErrors || {}
         const attachComponentMeta = component => {

packages/server/src/api/controllers/static/index.ts

@@ -339,10 +339,13 @@ export const getSignedUploadURL = async function (
       ctx.throw(400, "bucket and key values are required")
     }
     try {
+      let endpoint = datasource?.config?.endpoint
+      if (endpoint && !utils.urlHasProtocol(endpoint)) {
+        endpoint = `https://${endpoint}`
+      }
       const s3 = new S3({
         region: awsRegion,
-        endpoint: datasource?.config?.endpoint || undefined,
-
+        endpoint: endpoint,
         credentials: {
           accessKeyId: datasource?.config?.accessKeyId as string,
           secretAccessKey: datasource?.config?.secretAccessKey as string,
@@ -350,8 +353,8 @@ export const getSignedUploadURL = async function (
       })
       const params = { Bucket: bucket, Key: key }
       signedUrl = await getSignedUrl(s3, new PutObjectCommand(params))
-      if (datasource?.config?.endpoint) {
-        publicUrl = `${datasource.config.endpoint}/${bucket}/${key}`
+      if (endpoint) {
+        publicUrl = `${endpoint}/${bucket}/${key}`
       } else {
         publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
       }

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -80,7 +80,6 @@
       // Set some flags so the app knows we're in the builder
       window["##BUDIBASE_IN_BUILDER##"] = true
       window["##BUDIBASE_APP_ID##"] = appId
-      window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
@@ -90,7 +89,6 @@
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
       window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
-      window["##BUDIBASE_LOCATION##"] = location
       window["##BUDIBASE_SNIPPETS##"] = snippets
       window['##BUDIBASE_COMPONENT_ERRORS##'] = componentErrors
 

packages/server/src/api/routes/backup.ts

@@ -2,12 +2,14 @@ import Router from "@koa/router"
 import * as controller from "../controllers/backup"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
+import ensureTenantAppOwnership from "../../middleware/ensureTenantAppOwnership"
 
 const router: Router = new Router()
 
 router.post(
   "/api/backups/export",
   authorized(permissions.BUILDER),
+  ensureTenantAppOwnership,
   controller.exportAppDump
 )
 

packages/server/src/integrations/s3.ts

@@ -7,7 +7,7 @@ import {
   ConnectionInfo,
 } from "@budibase/types"
 
-import { S3 } from "@aws-sdk/client-s3"
+import { S3, S3ClientConfig } from "@aws-sdk/client-s3"
 import csv from "csvtojson"
 import stream from "stream"
 
@@ -157,13 +157,20 @@ const SCHEMA: Integration = {
 }
 
 class S3Integration implements IntegrationBase {
-  private readonly config: S3Config
-  private client
+  private readonly config: S3ClientConfig
+  private client: S3
 
   constructor(config: S3Config) {
-    this.config = config
-    if (this.config.endpoint) {
-      this.config.s3ForcePathStyle = true
+    this.config = {
+      forcePathStyle: config.s3ForcePathStyle || true,
+      credentials: {
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
+      },
+      region: config.region,
+    }
+    if (config.endpoint) {
+      this.config.forcePathStyle = true
     } else {
       delete this.config.endpoint
     }
@@ -176,7 +183,9 @@ class S3Integration implements IntegrationBase {
       connected: false,
     }
     try {
-      await this.client.listBuckets()
+      await this.client.listBuckets({
+        MaxBuckets: 1,
+      })
       response.connected = true
     } catch (e: any) {
       response.error = e.message as string
@@ -253,7 +262,7 @@ class S3Integration implements IntegrationBase {
         .on("error", () => {
           csvError = true
         })
-      fileStream.on("finish", () => {
+      fileStream.on("end", () => {
         resolve(response)
       })
     }).catch(err => {

packages/server/src/middleware/ensureTenantAppOwnership.ts

@@ -0,0 +1,16 @@
+import { tenancy, utils } from "@budibase/backend-core"
+import { UserCtx } from "@budibase/types"
+
+async function ensureTenantAppOwnership(ctx: UserCtx, next: any) {
+  const appId = await utils.getAppIdFromCtx(ctx)
+  if (!appId) {
+    ctx.throw(400, "appId must be provided")
+  }
+  const tenantId = tenancy.getTenantId()
+  if (appId !== tenantId) {
+    ctx.throw(403, `App does not belong to tenant`)
+  }
+  await next()
+}
+
+export default ensureTenantAppOwnership

packages/server/src/middleware/tests/ensureTenantAppOwnership.spec.js

@@ -0,0 +1,75 @@
+import ensureTenantAppOwnership from "../ensureTenantAppOwnership"
+import { tenancy, utils } from "@budibase/backend-core"
+
+jest.mock("@budibase/backend-core", () => ({
+  tenancy: {
+    getTenantId: jest.fn(),
+  },
+  utils: {
+    getAppIdFromCtx: jest.fn(),
+  },
+}))
+
+class TestConfiguration {
+  constructor(appId = "tenant_1") {
+    this.next = jest.fn()
+    this.throw = jest.fn()
+    this.middleware = ensureTenantAppOwnership
+
+    this.ctx = {
+      next: this.next,
+      throw: this.throw,
+    }
+
+    utils.getAppIdFromCtx.mockResolvedValue(appId)
+  }
+
+  async executeMiddleware() {
+    return this.middleware(this.ctx, this.next)
+  }
+
+  afterEach() {
+    jest.clearAllMocks()
+  }
+}
+
+describe("Ensure Tenant Ownership Middleware", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.afterEach()
+  })
+
+  it("calls next() when appId matches tenant ID", async () => {
+    tenancy.getTenantId.mockReturnValue("tenant_1")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.next).toHaveBeenCalled()
+  })
+
+  it("throws 403 when appId does not match tenant ID", async () => {
+    tenancy.getTenantId.mockReturnValue("tenant_2")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.throw).toHaveBeenCalledWith(
+      403,
+      "App does not belong to tenant"
+    )
+  })
+
+  it("throws 400 when appId is missing", async () => {
+    utils.getAppIdFromCtx.mockResolvedValue(null)
+
+    await config.executeMiddleware()
+
+    expect(config.throw).toHaveBeenCalledWith(400, "appId must be provided")
+  })
+})

packages/server/src/sdk/app/datasources/datasources.ts

@@ -120,7 +120,7 @@ export function areRESTVariablesValid(datasource: Datasource) {
 
 export function checkDatasourceTypes(schema: Integration, config: any) {
   for (let key of Object.keys(config)) {
-    if (!schema.datasource[key]) {
+    if (!schema.datasource?.[key]) {
       continue
     }
     const type = schema.datasource[key].type
@@ -149,7 +149,9 @@ async function enrichDatasourceWithValues(
   ) as Datasource
   processed.entities = entities
   const definition = await getDefinition(processed.source)
-  processed.config = checkDatasourceTypes(definition!, processed.config)
+  if (definition) {
+    processed.config = checkDatasourceTypes(definition, processed.config)
+  }
   return {
     datasource: processed,
     envVars: env as Record<string, string>,

packages/types/src/sdk/datasources.ts

@@ -157,7 +157,7 @@ export interface Integration {
   friendlyName: string
   type?: string
   iconUrl?: string
-  datasource: DatasourceConfig
+  datasource?: DatasourceConfig
   query: {
     [key: string]: QueryDefinition
   }

packages/types/src/ui/stores/misc.ts

@@ -1,2 +1,8 @@
 // type purely to capture structures that the type is unknown, but maybe known later
 export type UIObject = Record<string, any>
+
+export const enum DropPosition {
+  ABOVE = "above",
+  BELOW = "below",
+  INSIDE = "inside",
+}