diff --git a/hosting/.env b/hosting/.env index 173d409d04..23681f1f57 100644 --- a/hosting/.env +++ b/hosting/.env @@ -19,7 +19,6 @@ MINIO_PORT=4004 COUCH_DB_PORT=4005 COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 -WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION SQL_MAX_ROWS= diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index 1f16baa9e2..057d51a887 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -74,7 +74,6 @@ services: - WORKER_UPSTREAM_URL=http://worker-service:4003 - MINIO_UPSTREAM_URL=http://minio-service:9000 - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984 - - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080 - RESOLVER=127.0.0.11 depends_on: - minio-service diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index c7a22eb2b3..ec24765149 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -87,7 +87,6 @@ services: - WORKER_UPSTREAM_URL=http://worker-service:4003 - MINIO_UPSTREAM_URL=http://minio-service:9000 - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984 - - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080 - RESOLVER=127.0.0.11 depends_on: - minio-service @@ -112,19 +111,6 @@ services: volumes: - redis_data:/data - watchtower-service: - restart: always - image: containrrr/watchtower - volumes: - - /var/run/docker.sock:/var/run/docker.sock - command: --debug --http-api-update bbapps bbworker bbproxy - environment: - - WATCHTOWER_HTTP_API=true - - WATCHTOWER_HTTP_API_TOKEN=budibase - - WATCHTOWER_CLEANUP=true - labels: - - "com.centurylinklabs.watchtower.enable=false" - volumes: couchdb3_data: driver: local diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml deleted file mode 100644 index d9f8384688..0000000000 --- a/hosting/envoy.yaml +++ /dev/null @@ -1,152 +0,0 @@ -static_resources: - listeners: - - name: main_listener - address: - socket_address: { address: 0.0.0.0, port_value: 10000 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress - codec_type: auto - route_config: - name: local_route - virtual_hosts: - - name: local_services - domains: ["*"] - routes: - - match: { prefix: "/app/" } - route: - cluster: app-service - prefix_rewrite: "/" - - - match: { path: "/v1/update" } - route: - cluster: watchtower-service - - - match: { prefix: "/builder/" } - route: - cluster: app-service - - - match: { prefix: "/builder" } - route: - cluster: app-service - - - match: { prefix: "/app_" } - route: - cluster: app-service - - # special cases for worker admin (deprecated), global and system API - - match: { prefix: "/api/global/" } - route: - cluster: worker-service - - - match: { prefix: "/api/admin/" } - route: - cluster: worker-service - - - match: { prefix: "/api/system/" } - route: - cluster: worker-service - - - match: { path: "/" } - route: - cluster: app-service - - # special case for when API requests are made, can just forward, not to minio - - match: { prefix: "/api/" } - route: - cluster: app-service - timeout: 120s - - - match: { prefix: "/worker/" } - route: - cluster: worker-service - prefix_rewrite: "/" - - - match: { prefix: "/db/" } - route: - cluster: couchdb-service - prefix_rewrite: "/" - - # minio is on the default route because this works - # best, minio + AWS SDK doesn't handle path proxy - - match: { prefix: "/" } - route: - cluster: minio-service - - http_filters: - - name: envoy.filters.http.router - - clusters: - - name: app-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: app-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: app-service - port_value: 4002 - - - name: minio-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: minio-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: minio-service - port_value: 9000 - - - name: worker-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: worker-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: worker-service - port_value: 4003 - - - name: couchdb-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: couchdb-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: couchdb-service - port_value: 5984 - - - name: watchtower-service - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: watchtower-service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: watchtower-service - port_value: 8080 - diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 6c1d9e5dbd..f63bb1941a 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -18,7 +18,6 @@ WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 REDIS_PORT=6379 -WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION # An admin user can be automatically created initially if these are set @@ -26,4 +25,4 @@ BB_ADMIN_USER_EMAIL= BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ -PLUGINS_DIR= \ No newline at end of file +PLUGINS_DIR= diff --git a/hosting/portainer/template.json b/hosting/portainer/template.json index 29107b674e..4ca5b5e94f 100644 --- a/hosting/portainer/template.json +++ b/hosting/portainer/template.json @@ -78,11 +78,6 @@ "default": "6379", "preset": true }, - { - "name": "WATCHTOWER_PORT", - "default": "6161", - "preset": true - }, { "name": "BUDIBASE_ENVIRONMENT", "default": "PRODUCTION", diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile index 42327be087..9ec458a219 100644 --- a/hosting/proxy/Dockerfile +++ b/hosting/proxy/Dockerfile @@ -22,5 +22,4 @@ ENV APPS_UPSTREAM_URL=http://app-service:4002 ENV WORKER_UPSTREAM_URL=http://worker-service:4003 ENV MINIO_UPSTREAM_URL=http://minio-service:9000 ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984 -ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080 ENV RESOLVER=127.0.0.11 diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 59722dac5c..f18bae09a4 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -81,7 +81,6 @@ http { set $worker ${WORKER_UPSTREAM_URL}; set $minio ${MINIO_UPSTREAM_URL}; set $couchdb ${COUCHDB_UPSTREAM_URL}; - set $watchtower ${WATCHTOWER_UPSTREAM_URL}; location /health { access_log off; @@ -107,10 +106,6 @@ http { proxy_pass $apps; } - location = /v1/update { - proxy_pass $watchtower; - } - location ~ ^/(builder|app_) { proxy_http_version 1.1; diff --git a/hosting/scripts/airgapped/airgappedDockerBuild.js b/hosting/scripts/airgapped/airgappedDockerBuild.js index 58bc7c09a9..432ea9a370 100755 --- a/hosting/scripts/airgapped/airgappedDockerBuild.js +++ b/hosting/scripts/airgapped/airgappedDockerBuild.js @@ -12,7 +12,6 @@ let IMAGES = { couch: "ibmcom/couchdb3", curl: "curlimages/curl", redis: "redis", - watchtower: "containrrr/watchtower", } if (IS_SINGLE_IMAGE) { @@ -53,4 +52,4 @@ if (!IS_SINGLE_IMAGE) { copyFile(FILES.ENV) // compress -execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`) \ No newline at end of file +execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`) diff --git a/lerna.json b/lerna.json index 76e2e349f8..19b603b1cd 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.1.1", + "version": "3.2.0", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 7bba9f23a4..de6b9cad8b 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -271,8 +271,8 @@ export const flags = new FlagSet({ [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true), [FeatureFlag.SQS]: Flag.boolean(true), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true), - [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), - [FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()), + [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true), + [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true), }) type UnwrapPromise = T extends Promise ? U : T diff --git a/packages/builder/src/pages/builder/portal/settings/version.svelte b/packages/builder/src/pages/builder/portal/settings/version.svelte index c3898b7861..032c077557 100644 --- a/packages/builder/src/pages/builder/portal/settings/version.svelte +++ b/packages/builder/src/pages/builder/portal/settings/version.svelte @@ -4,12 +4,10 @@ Layout, Heading, Body, - Button, Divider, notifications, Label, - Modal, - ModalContent, + Link, } from "@budibase/bbui" import { API } from "api" import { auth, admin } from "stores/portal" @@ -21,8 +19,6 @@ let githubVersion let githubPublishedDate let githubPublishedTime - let needsUpdate = true - let updateModal // Only admins allowed here $: { @@ -31,21 +27,6 @@ } } - async function updateBudibase() { - try { - notifications.info("Updating budibase..") - await fetch("/v1/update", { - headers: { - Authorization: "Bearer budibase", - }, - }) - notifications.success("Your budibase installation is up to date.") - getVersion() - } catch (err) { - notifications.error(`Error installing budibase update ${err}`) - } - } - async function getVersion() { try { version = await API.getBudibaseVersion() @@ -69,13 +50,6 @@ githubPublishedDate = new Date(githubResponse.published_at) githubPublishedTime = githubPublishedDate.toLocaleTimeString() githubPublishedDate = githubPublishedDate.toLocaleDateString() - - //Does Budibase need to be updated? - if (githubVersion === version) { - needsUpdate = false - } else { - needsUpdate = true - } } catch (error) { notifications.error("Error getting the latest Budibase version") githubVersion = null @@ -115,23 +89,15 @@ > -
- + Updating Budibase + To update your self-host installation, follow the docs found here. - - - Are you sure you want to update your budibase installation to the - latest version? - - -
+ {/if} {/if} diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 379927342a..9c5313e9da 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -355,6 +355,93 @@ describe("Loop automations", () => { expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length) }) + it("should run an automation with a loop and update row step using stepIds", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const rows = [ + { name: "Row 1", value: 1, tableId: table._id }, + { name: "Row 2", value: 2, tableId: table._id }, + { name: "Row 3", value: 3, tableId: table._id }, + ] + + await config.api.row.bulkImport(table._id!, { rows }) + + const builder = createAutomationBuilder({ + name: "Test Loop and Update Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + }, + { stepId: "abc123" } + ) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.abc123.rows }}", + }) + .updateRow({ + rowId: "{{ loop.currentItem._id }}", + row: { + name: "Updated {{ loop.currentItem.name }}", + value: "{{ loop.currentItem.value }}", + tableId: table._id, + }, + meta: {}, + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + const expectedRows = [ + { name: "Updated Row 1", value: 1 }, + { name: "Updated Row 2", value: 2 }, + { name: "Updated Row 3", value: 3 }, + ] + + expect(results.steps[1].outputs.items).toEqual( + expect.arrayContaining( + expectedRows.map(row => + expect.objectContaining({ + success: true, + row: expect.objectContaining(row), + }) + ) + ) + ) + + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining( + expectedRows.map(row => expect.objectContaining(row)) + ) + ) + + expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length) + expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length) + }) + it("should run an automation with a loop and delete row step", async () => { const table = await config.createTable({ name: "TestTable", diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index d13be8d410..91bf4a211f 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -385,7 +385,7 @@ class Orchestrator { stepIdx: number, pathIdx?: number ): Promise { - await processObject(loopStep.inputs, this.context) + await processObject(loopStep.inputs, this.processContext(this.context)) const iterations = getLoopIterations(loopStep) let stepToLoopIndex = stepIdx + 1 let pathStepIdx = (pathIdx || stepIdx) + 1 diff --git a/packages/shared-core/src/helpers/async.ts b/packages/shared-core/src/helpers/async.ts new file mode 100644 index 0000000000..d9f64ac825 --- /dev/null +++ b/packages/shared-core/src/helpers/async.ts @@ -0,0 +1,3 @@ +export async function wait(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts index 7603a9b88b..0aa378b46a 100644 --- a/packages/shared-core/src/helpers/index.ts +++ b/packages/shared-core/src/helpers/index.ts @@ -1,5 +1,7 @@ export * from "./helpers" export * from "./integrations" +export * from "./async" +export * from "./retry" export * as cron from "./cron" export * as schema from "./schema" export * as views from "./views" diff --git a/packages/shared-core/src/helpers/retry.ts b/packages/shared-core/src/helpers/retry.ts new file mode 100644 index 0000000000..04494da227 --- /dev/null +++ b/packages/shared-core/src/helpers/retry.ts @@ -0,0 +1,28 @@ +import { wait } from "./async" + +interface RetryOpts { + times?: number +} + +export async function retry( + fn: () => Promise, + opts?: RetryOpts +): Promise { + const { times = 3 } = opts || {} + if (times < 1) { + throw new Error(`invalid retry count: ${times}`) + } + + let lastError: any + for (let i = 0; i < times; i++) { + const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5) + await wait(backoff) + + try { + return await fn() + } catch (e) { + lastError = e + } + } + throw lastError +} diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index b6352ea5e7..48ab2b586f 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types" import env from "../../../environment" import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core" import nodeFetch from "node-fetch" +import { helpers } from "@budibase/shared-core" let sqsAvailable: boolean async function isSqsAvailable() { @@ -12,17 +13,22 @@ async function isSqsAvailable() { } try { - const couchInfo = dbCore.getCouchInfo() - if (!couchInfo.sqlUrl) { + const { url } = dbCore.getCouchInfo() + if (!url) { sqsAvailable = false return false } - await nodeFetch(couchInfo.sqlUrl, { - timeout: 1000, - }) + await helpers.retry( + async () => { + await nodeFetch(url, { timeout: 2000 }) + }, + { times: 3 } + ) + console.log("connected to SQS") sqsAvailable = true return true } catch (e) { + console.warn("failed to connect to SQS", e) sqsAvailable = false return false }