Merge branch 'master' into BUDI-8683/correct-automation-cron-server-timestamp

hosting/.env

@@ -19,7 +19,6 @@ MINIO_PORT=4004
 COUCH_DB_PORT=4005
 COUCH_DB_SQS_PORT=4006
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 SQL_MAX_ROWS=
 

hosting/docker-compose.build.yaml

@@ -74,7 +74,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service

hosting/docker-compose.yaml

@@ -87,7 +87,6 @@ services:
       - WORKER_UPSTREAM_URL=http://worker-service:4003
       - MINIO_UPSTREAM_URL=http://minio-service:9000
       - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
       - RESOLVER=127.0.0.11
     depends_on:
       - minio-service
@@ -112,19 +111,6 @@ services:
     volumes:
       - redis_data:/data
 
-  watchtower-service:
-    restart: always
-    image: containrrr/watchtower
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: --debug --http-api-update bbapps bbworker bbproxy
-    environment:
-      - WATCHTOWER_HTTP_API=true
-      - WATCHTOWER_HTTP_API_TOKEN=budibase
-      - WATCHTOWER_CLEANUP=true
-    labels:
-      - "com.centurylinklabs.watchtower.enable=false"
-
 volumes:
   couchdb3_data:
     driver: local

hosting/envoy.yaml

@@ -1,152 +0,0 @@
-static_resources:
-  listeners:
-  - name: main_listener
-    address:
-      socket_address: { address: 0.0.0.0, port_value: 10000 }
-    filter_chains:
-    - filters:
-      - name: envoy.filters.network.http_connection_manager
-        typed_config:
-          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
-          stat_prefix: ingress
-          codec_type: auto
-          route_config:
-            name: local_route
-            virtual_hosts:
-            - name: local_services
-              domains: ["*"]
-              routes:
-              - match: { prefix: "/app/" }
-                route:
-                  cluster: app-service
-                  prefix_rewrite: "/"
-
-              - match: { path: "/v1/update" }
-                route:
-                  cluster: watchtower-service
-
-              - match: { prefix: "/builder/" }
-                route:
-                  cluster: app-service
-
-              - match: { prefix: "/builder" }
-                route:
-                  cluster: app-service
-              
-              - match: { prefix: "/app_" }
-                route:
-                  cluster: app-service
-
-              # special cases for worker admin (deprecated), global and system API
-              - match: { prefix: "/api/global/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/admin/" }
-                route:
-                  cluster: worker-service
-
-              - match: { prefix: "/api/system/" }
-                route:
-                  cluster: worker-service
-
-              - match: { path: "/" }
-                route:
-                  cluster: app-service
-
-              # special case for when API requests are made, can just forward, not to minio    
-              - match: { prefix: "/api/" }
-                route:
-                  cluster: app-service
-                  timeout: 120s
-
-              - match: { prefix: "/worker/" }
-                route: 
-                  cluster: worker-service
-                  prefix_rewrite: "/"
-
-              - match: { prefix: "/db/" }
-                route:
-                  cluster: couchdb-service
-                  prefix_rewrite: "/"
-
-              # minio is on the default route because this works
-              # best, minio + AWS SDK doesn't handle path proxy
-              - match: { prefix: "/" }
-                route:
-                  cluster: minio-service
-
-          http_filters:
-          - name: envoy.filters.http.router
-
-  clusters:
-  - name: app-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: app-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: app-service
-                port_value: 4002
-
-  - name: minio-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: minio-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: minio-service
-                port_value: 9000
-
-  - name: worker-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: worker-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: worker-service
-                port_value: 4003
-
-  - name: couchdb-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: couchdb-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address:
-                address: couchdb-service
-                port_value: 5984
-
-  - name: watchtower-service
-    connect_timeout: 0.25s
-    type: strict_dns
-    lb_policy: round_robin
-    load_assignment:
-      cluster_name: watchtower-service
-      endpoints:
-      - lb_endpoints:
-        - endpoint:
-            address:
-              socket_address: 
-                address: watchtower-service
-                port_value: 8080
-

hosting/hosting.properties

@@ -18,7 +18,6 @@ WORKER_PORT=4003
 MINIO_PORT=4004
 COUCH_DB_PORT=4005
 REDIS_PORT=6379
-WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
 
 # An admin user can be automatically created initially if these are set

hosting/portainer/template.json

@@ -78,11 +78,6 @@
           "default": "6379",
           "preset": true
         },
-        {
-          "name": "WATCHTOWER_PORT",
-          "default": "6161",
-          "preset": true
-        },
         {
           "name": "BUDIBASE_ENVIRONMENT",
           "default": "PRODUCTION",

hosting/proxy/Dockerfile

@@ -22,5 +22,4 @@ ENV APPS_UPSTREAM_URL=http://app-service:4002
 ENV WORKER_UPSTREAM_URL=http://worker-service:4003
 ENV MINIO_UPSTREAM_URL=http://minio-service:9000
 ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
-ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
 ENV RESOLVER=127.0.0.11

hosting/proxy/nginx.prod.conf

@@ -81,7 +81,6 @@ http {
     set $worker ${WORKER_UPSTREAM_URL};
     set $minio ${MINIO_UPSTREAM_URL};
     set $couchdb ${COUCHDB_UPSTREAM_URL};
-    set $watchtower ${WATCHTOWER_UPSTREAM_URL};
 
     location /health {
       access_log off;
@@ -107,10 +106,6 @@ http {
       proxy_pass      $apps;
     }
 
-    location = /v1/update {
-      proxy_pass      $watchtower;
-    }
-
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
 

hosting/scripts/airgapped/airgappedDockerBuild.js

@@ -12,7 +12,6 @@ let IMAGES = {
 	couch: "ibmcom/couchdb3",
 	curl: "curlimages/curl",
 	redis: "redis",
-	watchtower: "containrrr/watchtower",
 }
 
 if (IS_SINGLE_IMAGE) {

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.1.1",
+  "version": "3.2.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/features/features.ts

@@ -271,8 +271,8 @@ export const flags = new FlagSet({
   [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
   [FeatureFlag.SQS]: Flag.boolean(true),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()),
+  [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
+  [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
 })
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -4,12 +4,10 @@
     Layout,
     Heading,
     Body,
-    Button,
     Divider,
     notifications,
     Label,
-    Modal,
-    ModalContent,
+    Link,
   } from "@budibase/bbui"
   import { API } from "api"
   import { auth, admin } from "stores/portal"
@@ -21,8 +19,6 @@
   let githubVersion
   let githubPublishedDate
   let githubPublishedTime
-  let needsUpdate = true
-  let updateModal
 
   // Only admins allowed here
   $: {
@@ -31,21 +27,6 @@
     }
   }
 
-  async function updateBudibase() {
-    try {
-      notifications.info("Updating budibase..")
-      await fetch("/v1/update", {
-        headers: {
-          Authorization: "Bearer budibase",
-        },
-      })
-      notifications.success("Your budibase installation is up to date.")
-      getVersion()
-    } catch (err) {
-      notifications.error(`Error installing budibase update ${err}`)
-    }
-  }
-
   async function getVersion() {
     try {
       version = await API.getBudibaseVersion()
@@ -69,13 +50,6 @@
       githubPublishedDate = new Date(githubResponse.published_at)
       githubPublishedTime = githubPublishedDate.toLocaleTimeString()
       githubPublishedDate = githubPublishedDate.toLocaleDateString()
-
-      //Does Budibase need to be updated?
-      if (githubVersion === version) {
-        needsUpdate = false
-      } else {
-        needsUpdate = true
-      }
     } catch (error) {
       notifications.error("Error getting the latest Budibase version")
       githubVersion = null
@@ -115,23 +89,15 @@
         >
       </Layout>
       <Divider />
-      <div>
-        <Button cta on:click={updateModal.show} disabled={!needsUpdate}
-          >Update Budibase</Button
+      <Layout noPadding gap="XS">
+        <Heading>Updating Budibase</Heading>
+        <Body
+          >To update your self-host installation, follow the docs found <Link
+            size="L"
+            href="https://docs.budibase.com/docs/updating-budibase">here.</Link
+          ></Body
         >
-        <Modal bind:this={updateModal}>
-          <ModalContent
-            title="Update Budibase"
-            confirmText="Update"
-            onConfirm={updateBudibase}
-          >
-            <span
-              >Are you sure you want to update your budibase installation to the
-              latest version?</span
-            >
-          </ModalContent>
-        </Modal>
-      </div>
+      </Layout>
     {/if}
   </Layout>
 {/if}

packages/server/src/automations/tests/scenarios/looping.spec.ts

@@ -355,6 +355,93 @@ describe("Loop automations", () => {
     expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length)
   })
 
+  it("should run an automation with a loop and update row step using stepIds", async () => {
+    const table = await config.createTable({
+      name: "TestTable",
+      type: "table",
+      schema: {
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+          constraints: {
+            presence: true,
+          },
+        },
+        value: {
+          name: "value",
+          type: FieldType.NUMBER,
+          constraints: {
+            presence: true,
+          },
+        },
+      },
+    })
+
+    const rows = [
+      { name: "Row 1", value: 1, tableId: table._id },
+      { name: "Row 2", value: 2, tableId: table._id },
+      { name: "Row 3", value: 3, tableId: table._id },
+    ]
+
+    await config.api.row.bulkImport(table._id!, { rows })
+
+    const builder = createAutomationBuilder({
+      name: "Test Loop and Update Row",
+    })
+
+    const results = await builder
+      .appAction({ fields: {} })
+      .queryRows(
+        {
+          tableId: table._id!,
+        },
+        { stepId: "abc123" }
+      )
+      .loop({
+        option: LoopStepType.ARRAY,
+        binding: "{{ steps.abc123.rows }}",
+      })
+      .updateRow({
+        rowId: "{{ loop.currentItem._id }}",
+        row: {
+          name: "Updated {{ loop.currentItem.name }}",
+          value: "{{ loop.currentItem.value }}",
+          tableId: table._id,
+        },
+        meta: {},
+      })
+      .queryRows({
+        tableId: table._id!,
+      })
+      .run()
+
+    const expectedRows = [
+      { name: "Updated Row 1", value: 1 },
+      { name: "Updated Row 2", value: 2 },
+      { name: "Updated Row 3", value: 3 },
+    ]
+
+    expect(results.steps[1].outputs.items).toEqual(
+      expect.arrayContaining(
+        expectedRows.map(row =>
+          expect.objectContaining({
+            success: true,
+            row: expect.objectContaining(row),
+          })
+        )
+      )
+    )
+
+    expect(results.steps[2].outputs.rows).toEqual(
+      expect.arrayContaining(
+        expectedRows.map(row => expect.objectContaining(row))
+      )
+    )
+
+    expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length)
+    expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length)
+  })
+
   it("should run an automation with a loop and delete row step", async () => {
     const table = await config.createTable({
       name: "TestTable",

packages/server/src/threads/automation.ts

@@ -385,7 +385,7 @@ class Orchestrator {
     stepIdx: number,
     pathIdx?: number
   ): Promise<number> {
-    await processObject(loopStep.inputs, this.context)
+    await processObject(loopStep.inputs, this.processContext(this.context))
     const iterations = getLoopIterations(loopStep)
     let stepToLoopIndex = stepIdx + 1
     let pathStepIdx = (pathIdx || stepIdx) + 1

packages/shared-core/src/helpers/async.ts

@@ -0,0 +1,3 @@
+export async function wait(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

packages/shared-core/src/helpers/index.ts

@@ -1,5 +1,7 @@
 export * from "./helpers"
 export * from "./integrations"
+export * from "./async"
+export * from "./retry"
 export * as cron from "./cron"
 export * as schema from "./schema"
 export * as views from "./views"

packages/shared-core/src/helpers/retry.ts

@@ -0,0 +1,28 @@
+import { wait } from "./async"
+
+interface RetryOpts {
+  times?: number
+}
+
+export async function retry<T>(
+  fn: () => Promise<T>,
+  opts?: RetryOpts
+): Promise<T> {
+  const { times = 3 } = opts || {}
+  if (times < 1) {
+    throw new Error(`invalid retry count: ${times}`)
+  }
+
+  let lastError: any
+  for (let i = 0; i < times; i++) {
+    const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5)
+    await wait(backoff)
+
+    try {
+      return await fn()
+    } catch (e) {
+      lastError = e
+    }
+  }
+  throw lastError
+}

packages/worker/src/api/controllers/system/environment.ts

@@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
 import env from "../../../environment"
 import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
+import { helpers } from "@budibase/shared-core"
 
 let sqsAvailable: boolean
 async function isSqsAvailable() {
@@ -12,17 +13,22 @@ async function isSqsAvailable() {
   }
 
   try {
-    const couchInfo = dbCore.getCouchInfo()
-    if (!couchInfo.sqlUrl) {
+    const { url } = dbCore.getCouchInfo()
+    if (!url) {
       sqsAvailable = false
       return false
     }
-    await nodeFetch(couchInfo.sqlUrl, {
-      timeout: 1000,
-    })
+    await helpers.retry(
+      async () => {
+        await nodeFetch(url, { timeout: 2000 })
+      },
+      { times: 3 }
+    )
+    console.log("connected to SQS")
     sqsAvailable = true
     return true
   } catch (e) {
+    console.warn("failed to connect to SQS", e)
     sqsAvailable = false
     return false
   }