Merge pull request #11667 from Budibase/merge-master-develop

Merging master -> develop

charts/budibase/templates/secrets.yaml

@@ -1,4 +1,5 @@
-{{- if .Values.globals.createSecrets -}}
+{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (include "budibase.fullname" .) }}
+{{- if .Values.globals.createSecrets }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -10,8 +11,15 @@ metadata:
     heritage: "{{ .Release.Service }}"
 type: Opaque
 data:
+  {{- if $existingSecret }}
+  internalApiKey: {{ index $existingSecret.data "internalApiKey" }}
+  jwtSecret: {{ index $existingSecret.data "jwtSecret" }}
+  objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }}
+  objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }}
+  {{- else }}
   internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }}
   jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }}
   objectStoreAccess: {{ template "budibase.defaultsecret" .Values.services.objectStore.accessKey }}
   objectStoreSecret: {{ template "budibase.defaultsecret" .Values.services.objectStore.secretKey }}
-{{- end -}}
+  {{- end }}
+{{- end }}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.9.33-alpha.16",
+  "version": "2.9.38",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
@@ -19,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -8,7 +8,6 @@ import {
   DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
-  DocExistsResponse,
   Document,
   isDocument,
 } from "@budibase/types"
@@ -121,19 +120,6 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
-  async docExists(docId: string): Promise<DocExistsResponse> {
-    const db = await this.checkSetup()
-    let _rev, exists
-    try {
-      const { etag } = await db.head(docId)
-      _rev = etag
-      exists = true
-    } catch (err) {
-      exists = false
-    }
-    return { _rev, exists }
-  }
-
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string

packages/builder/src/components/backend/DataTable/ViewDataTable.svelte

@@ -10,6 +10,7 @@
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
   import HideAutocolumnButton from "./buttons/HideAutocolumnButton.svelte"
   import { notifications } from "@budibase/bbui"
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
 
   export let view = {}
 
@@ -19,6 +20,14 @@
   let type = "internal"
 
   $: name = view.name
+  $: calculation = view.calculation
+
+  $: supportedFormats = Object.values(ROW_EXPORT_FORMATS).filter(key => {
+    if (calculation && key === ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA) {
+      return false
+    }
+    return true
+  })
 
   // Fetch rows for specified view
   $: fetchViewData(name, view.field, view.groupBy, view.calculation)
@@ -68,5 +77,5 @@
   {/if}
   <ManageAccessButton resourceId={decodeURI(name)} />
   <HideAutocolumnButton bind:hideAutocolumns />
-  <ExportButton view={view.name} />
+  <ExportButton view={view.name} formats={supportedFormats} />
 </Table>

packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte

@@ -7,6 +7,7 @@
   export let sorting
   export let disabled = false
   export let selectedRows
+  export let formats
 
   let modal
 </script>
@@ -15,5 +16,5 @@
   Export
 </ActionButton>
 <Modal bind:this={modal}>
-  <ExportModal {view} {filters} {sorting} {selectedRows} />
+  <ExportModal {view} {filters} {sorting} {selectedRows} {formats} />
 </Modal>

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -9,30 +9,43 @@
   import download from "downloadjs"
   import { API } from "api"
   import { Constants, LuceneUtils } from "@budibase/frontend-core"
-
+  import { ROW_EXPORT_FORMATS } from "constants/backend"
-  const FORMATS = [
-    {
-      name: "CSV",
-      key: "csv",
-    },
-    {
-      name: "JSON",
-      key: "json",
-    },
-    {
-      name: "JSON with Schema",
-      key: "jsonWithSchema",
-    },
-  ]
 
   export let view
   export let filters
   export let sorting
   export let selectedRows = []
+  export let formats
 
-  let exportFormat = FORMATS[0].key
+  const FORMATS = [
+    {
+      name: "CSV",
+      key: ROW_EXPORT_FORMATS.CSV,
+    },
+    {
+      name: "JSON",
+      key: ROW_EXPORT_FORMATS.JSON,
+    },
+    {
+      name: "JSON with Schema",
+      key: ROW_EXPORT_FORMATS.JSON_WITH_SCHEMA,
+    },
+  ]
+
+  $: options = FORMATS.filter(format => {
+    if (formats && !formats.includes(format.key)) {
+      return false
+    }
+    return true
+  })
+
+  let exportFormat
   let filterLookup
 
+  $: if (options && !exportFormat) {
+    exportFormat = Array.isArray(options) ? options[0]?.key : []
+  }
+
   $: luceneFilter = LuceneUtils.buildLuceneQuery(filters)
   $: exportOpDisplay = buildExportOpDisplay(sorting, filterDisplay, filters)
 
@@ -190,7 +203,7 @@
   <Select
     label="Format"
     bind:value={exportFormat}
-    options={FORMATS}
+    {options}
     placeholder={null}
     getOptionLabel={x => x.name}
     getOptionValue={x => x.key}

packages/builder/src/constants/backend/index.js

@@ -287,3 +287,9 @@ export const DatasourceTypes = {
   GRAPH: "Graph",
   API: "API",
 }
+
+export const ROW_EXPORT_FORMATS = {
+  CSV: "csv",
+  JSON: "json",
+  JSON_WITH_SCHEMA: "jsonWithSchema",
+}

packages/client/src/components/app/forms/InnerForm.svelte

@@ -136,7 +136,7 @@
     // Check arrays - remove any values not present in the field schema and
     // convert any values supplied to strings
     if (Array.isArray(value) && type === "array" && schema) {
-      const options = schema?.constraints.inclusion || []
+      const options = schema?.constraints?.inclusion || []
       return value.map(opt => String(opt)).filter(opt => options.includes(opt))
     }
     return value

packages/server/src/api/controllers/application.ts

@@ -39,8 +39,9 @@ import {
 } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream } from "../../utilities"
-import { doesUserHaveLock } from "../../utilities/redis"
+import { doesUserHaveLock, getLocksById } from "../../utilities/redis"
 import { cleanupAutomations } from "../../automations/utils"
+import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { groups, licensing, quotas } from "@budibase/pro"
 import {
@@ -50,6 +51,7 @@ import {
   PlanType,
   Screen,
   UserCtx,
+  ContextUser,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"

packages/server/src/api/controllers/automation.ts

@@ -20,7 +20,7 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationResults,
-  Ctx,
+  BBContext,
 } from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
 import sdk from "../../sdk"
@@ -73,7 +73,7 @@ function cleanAutomationInputs(automation: Automation) {
   return automation
 }
 
-export async function create(ctx: Ctx) {
+export async function create(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -142,7 +142,7 @@ export async function handleStepEvents(
   }
 }
 
-export async function update(ctx: Ctx) {
+export async function update(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -193,7 +193,7 @@ export async function update(ctx: Ctx) {
   builderSocket?.emitAutomationUpdate(ctx, automation)
 }
 
-export async function fetch(ctx: Ctx) {
+export async function fetch(ctx: BBContext) {
   const db = context.getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
@@ -203,11 +203,12 @@ export async function fetch(ctx: Ctx) {
   ctx.body = response.rows.map(row => row.doc)
 }
 
-export async function find(ctx: Ctx) {
+export async function find(ctx: BBContext) {
-  ctx.body = await sdk.automations.get(ctx.params.id)
+  const db = context.getAppDB()
+  ctx.body = await db.get(ctx.params.id)
 }
 
-export async function destroy(ctx: Ctx) {
+export async function destroy(ctx: BBContext) {
   const db = context.getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get<Automation>(automationId)
@@ -221,11 +222,11 @@ export async function destroy(ctx: Ctx) {
   builderSocket?.emitAutomationDeletion(ctx, automationId)
 }
 
-export async function logSearch(ctx: Ctx) {
+export async function logSearch(ctx: BBContext) {
   ctx.body = await automations.logs.logSearch(ctx.request.body)
 }
 
-export async function clearLogError(ctx: Ctx) {
+export async function clearLogError(ctx: BBContext) {
   const { automationId, appId } = ctx.request.body
   await context.doInAppContext(appId, async () => {
     const db = context.getProdAppDB()
@@ -244,15 +245,15 @@ export async function clearLogError(ctx: Ctx) {
   })
 }
 
-export async function getActionList(ctx: Ctx) {
+export async function getActionList(ctx: BBContext) {
   ctx.body = await getActionDefinitions()
 }
 
-export async function getTriggerList(ctx: Ctx) {
+export async function getTriggerList(ctx: BBContext) {
   ctx.body = getTriggerDefinitions()
 }
 
-export async function getDefinitionList(ctx: Ctx) {
+export async function getDefinitionList(ctx: BBContext) {
   ctx.body = {
     trigger: getTriggerDefinitions(),
     action: await getActionDefinitions(),
@@ -265,7 +266,7 @@ export async function getDefinitionList(ctx: Ctx) {
  *                   *
  *********************/
 
-export async function trigger(ctx: Ctx) {
+export async function trigger(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
 
@@ -310,7 +311,7 @@ function prepareTestInput(input: any) {
   return input
 }
 
-export async function test(ctx: Ctx) {
+export async function test(ctx: BBContext) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
   await setTestFlag(automation._id!)

packages/server/src/api/controllers/row/index.ts

@@ -95,7 +95,7 @@ export async function fetchView(ctx: any) {
     () =>
       sdk.rows.fetchView(tableId, viewName, {
         calculation,
-        group,
+        group: calculation ? group : null,
         field,
       }),
     {

packages/server/src/api/controllers/view/exporters.ts

@@ -27,7 +27,7 @@ export function json(rows: Row[]) {
 export function jsonWithSchema(schema: TableSchema, rows: Row[]) {
   const newSchema: TableSchema = {}
   Object.values(schema).forEach(column => {
-    if (!column.autocolumn) {
+    if (!column.autocolumn && column.name) {
       newSchema[column.name] = column
     }
   })

packages/server/src/automations/triggers.ts

@@ -6,11 +6,11 @@ import { isDevAppID } from "../db/utils"
 // need this to call directly, so we can get a response
 import { automationQueue } from "./bullboard"
 import { checkTestFlag } from "../utilities/redis"
+import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
 import { executeSynchronously } from "../threads/automation"
-import sdk from "../sdk"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -142,7 +142,7 @@ export async function rebootTrigger() {
       let automations = await getAllAutomations()
       let rebootEvents = []
       for (let automation of automations) {
-        if (sdk.automations.isReboot(automation)) {
+        if (utils.isRebootTrigger(automation)) {
           const job = {
             automation,
             event: {

packages/server/src/automations/utils.ts

@@ -17,17 +17,16 @@ import {
 import sdk from "../sdk"
 import { automationsEnabled } from "../features"
 
+const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
+const CRON_STEP_ID = definitions.CRON.stepId
 let Runner: Thread
 if (automationsEnabled()) {
   Runner = new Thread(ThreadType.AUTOMATION)
 }
 
-function loggingArgs(
+function loggingArgs(job: AutomationJob) {
-  job: AutomationJob,
+  return [
-  timing?: { start: number; complete?: boolean }
-) {
-  const logs: any[] = [
     {
       _logKey: "automation",
       trigger: job.data.automation.definition.trigger.event,
@@ -37,53 +36,24 @@ function loggingArgs(
       jobId: job.id,
     },
   ]
-  if (timing?.start) {
-    logs.push({
-      _logKey: "startTime",
-      start: timing.start,
-    })
-  }
-  if (timing?.start && timing?.complete) {
-    const end = new Date().getTime()
-    const duration = end - timing.start
-    logs.push({
-      _logKey: "endTime",
-      end,
-    })
-    logs.push({
-      _logKey: "duration",
-      duration,
-    })
-  }
-  return logs
 }
 
 export async function processEvent(job: AutomationJob) {
   const appId = job.data.event.appId!
   const automationId = job.data.automation._id!
-  const start = new Date().getTime()
   const task = async () => {
     try {
       // need to actually await these so that an error can be captured properly
-      console.log("automation running", ...loggingArgs(job, { start }))
+      console.log("automation running", ...loggingArgs(job))
 
       const runFn = () => Runner.run(job)
       const result = await quotas.addAutomation(runFn, {
         automationId,
       })
-      const end = new Date().getTime()
+      console.log("automation completed", ...loggingArgs(job))
-      const duration = end - start
-      console.log(
-        "automation completed",
-        ...loggingArgs(job, { start, complete: true })
-      )
       return result
     } catch (err) {
-      console.error(
+      console.error(`automation was unable to run`, err, ...loggingArgs(job))
-        `automation was unable to run`,
-        err,
-        ...loggingArgs(job, { start, complete: true })
-      )
       return { err }
     }
   }
@@ -163,6 +133,19 @@ export async function clearMetadata() {
   await db.bulkDocs(automationMetadata)
 }
 
+export function isCronTrigger(auto: Automation) {
+  return (
+    auto &&
+    auto.definition.trigger &&
+    auto.definition.trigger.stepId === CRON_STEP_ID
+  )
+}
+
+export function isRebootTrigger(auto: Automation) {
+  const trigger = auto ? auto.definition.trigger : null
+  return isCronTrigger(auto) && trigger?.inputs.cron === REBOOT_CRON
+}
+
 /**
  * This function handles checking of any cron jobs that need to be enabled/updated.
  * @param {string} appId The ID of the app in which we are checking for webhooks
@@ -170,14 +153,14 @@ export async function clearMetadata() {
  */
 export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
-  const validCron = sdk.automations.isCron(automation) && trigger?.inputs.cron
-  const needsCreated =
-    !sdk.automations.isReboot(automation) &&
-    !sdk.automations.disabled(automation)
   let enabled = false
 
   // need to create cron job
-  if (validCron && needsCreated) {
+  if (
+    isCronTrigger(automation) &&
+    !isRebootTrigger(automation) &&
+    trigger?.inputs.cron
+  ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
     const job: any = await automationQueue.add(

packages/server/src/sdk/app/automations/automations.ts

@@ -1,38 +0,0 @@
-import { context } from "@budibase/backend-core"
-import { Automation, AutomationState, DocumentType } from "@budibase/types"
-import { definitions } from "../../../automations/triggerInfo"
-
-const REBOOT_CRON = "@reboot"
-
-export async function exists(automationId: string) {
-  if (!automationId?.startsWith(DocumentType.AUTOMATION)) {
-    throw new Error("Invalid automation ID.")
-  }
-  const db = context.getAppDB()
-  return db.docExists(automationId)
-}
-
-export async function get(automationId: string) {
-  const db = context.getAppDB()
-  return (await db.get(automationId)) as Automation
-}
-
-export function disabled(automation: Automation) {
-  return automation.state === AutomationState.DISABLED || !hasSteps(automation)
-}
-
-export function isCron(automation: Automation) {
-  return (
-    automation?.definition.trigger &&
-    automation?.definition.trigger.stepId === definitions.CRON.stepId
-  )
-}
-
-export function isReboot(automation: Automation) {
-  const trigger = automation?.definition.trigger
-  return isCron(automation) && trigger?.inputs.cron === REBOOT_CRON
-}
-
-export function hasSteps(automation: Automation) {
-  return automation?.definition?.steps?.length > 0
-}

packages/server/src/sdk/app/automations/index.ts

@@ -1,9 +1,7 @@
 import * as webhook from "./webhook"
 import * as utils from "./utils"
-import * as automations from "./automations"
 
 export default {
   webhook,
   utils,
-  ...automations,
 }

packages/server/src/threads/automation.ts

@@ -1,6 +1,5 @@
 import { default as threadUtils } from "./utils"
 import { Job } from "bull"
-threadUtils.threadSetup()
 import {
   disableCronById,
   isErrorInOutput,
@@ -35,8 +34,8 @@ import { cloneDeep } from "lodash/fp"
 import { performance } from "perf_hooks"
 import * as sdkUtils from "../sdk/utils"
 import env from "../environment"
-import sdk from "../sdk"
 
+threadUtils.threadSetup()
 const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
@@ -520,8 +519,7 @@ class Orchestrator {
 
 export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   const appId = job.data.event.appId
-  const automation = job.data.automation
+  const automationId = job.data.automation._id
-  const automationId = automation._id
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
@@ -532,30 +530,10 @@ export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
     appId,
     automationId,
     task: async () => {
-      let automation = job.data.automation,
-        isCron = sdk.automations.isCron(job.data.automation),
-        notFound = false
-      try {
-        automation = await sdk.automations.get(automationId)
-      } catch (err: any) {
-        // automation no longer exists
-        notFound = err
-      }
-      const disabled = sdk.automations.disabled(automation)
-      const stopAutomation = disabled || notFound
       const envVars = await sdkUtils.getEnvironmentVariables()
       // put into automation thread for whole context
       await context.doInEnvironmentContext(envVars, async () => {
         const automationOrchestrator = new Orchestrator(job)
-        // hard stop on automations
-        if (isCron && stopAutomation) {
-          await automationOrchestrator.stopCron(
-            disabled ? "disabled" : "not_found"
-          )
-        }
-        if (stopAutomation) {
-          return
-        }
         try {
           const response = await automationOrchestrator.execute()
           callback(null, response)

packages/types/src/documents/app/automation.ts

@@ -100,10 +100,6 @@ export const AutomationStepIdArray = [
   ...Object.values(AutomationTriggerStepId),
 ]
 
-export enum AutomationState {
-  DISABLED = "disabled",
-}
-
 export interface Automation extends Document {
   definition: {
     steps: AutomationStep[]
@@ -116,7 +112,6 @@ export interface Automation extends Document {
   name: string
   internal?: boolean
   type?: string
-  state?: AutomationState
 }
 
 interface BaseIOStructure {

packages/types/src/sdk/db.ts

@@ -40,11 +40,6 @@ export type DatabasePutOpts = {
   force?: boolean
 }
 
-export type DocExistsResponse = {
-  _rev?: string
-  exists: boolean
-}
-
 export type DatabaseCreateIndexOpts = {
   index: {
     fields: string[]
@@ -95,7 +90,6 @@ export interface Database {
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
   get<T>(id?: string): Promise<T>
-  docExists(id: string): Promise<DocExistsResponse>
   remove(
     id: string | Document,
     rev?: string