MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8683/correct-automation-cron-server-timestamp

This commit is contained in:
Adria Navarro 2024-11-07 10:58:35 +01:00 committed by GitHub
parent 0667dd9d30 1b491715ae
commit 5089528ffd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 146 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hosting/.env
View File

@ -19,7 +19,6 @@ MINIO_PORT=4004
COUCH_DB_PORT=4005
COUCH_DB_SQS_PORT=4006
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
SQL_MAX_ROWS=

1
hosting/docker-compose.build.yaml
View File

@ -74,7 +74,6 @@ services:
- WORKER_UPSTREAM_URL=http://worker-service:4003
- MINIO_UPSTREAM_URL=http://minio-service:9000
- COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
- WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
- RESOLVER=127.0.0.11
depends_on:
- minio-service

14
hosting/docker-compose.yaml
View File

@ -87,7 +87,6 @@ services:
- WORKER_UPSTREAM_URL=http://worker-service:4003
- MINIO_UPSTREAM_URL=http://minio-service:9000
- COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
- WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
- RESOLVER=127.0.0.11
depends_on:
- minio-service
@ -112,19 +111,6 @@ services:
volumes:
- redis_data:/data
watchtower-service:
restart: always
image: containrrr/watchtower
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: --debug --http-api-update bbapps bbworker bbproxy
environment:
- WATCHTOWER_HTTP_API=true
- WATCHTOWER_HTTP_API_TOKEN=budibase
- WATCHTOWER_CLEANUP=true
labels:
- "com.centurylinklabs.watchtower.enable=false"
volumes:
couchdb3_data:
driver: local

152
hosting/envoy.yaml
View File

@ -1,152 +0,0 @@
static_resources:
listeners:
- name: main_listener
address:
socket_address: { address: 0.0.0.0, port_value: 10000 }
filter_chains:
- filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: ingress
codec_type: auto
route_config:
name: local_route
virtual_hosts:
- name: local_services
domains: ["*"]
routes:
- match: { prefix: "/app/" }
route:
cluster: app-service
prefix_rewrite: "/"
- match: { path: "/v1/update" }
route:
cluster: watchtower-service
- match: { prefix: "/builder/" }
route:
cluster: app-service
- match: { prefix: "/builder" }
route:
cluster: app-service
- match: { prefix: "/app_" }
route:
cluster: app-service
# special cases for worker admin (deprecated), global and system API
- match: { prefix: "/api/global/" }
route:
cluster: worker-service
- match: { prefix: "/api/admin/" }
route:
cluster: worker-service
- match: { prefix: "/api/system/" }
route:
cluster: worker-service
- match: { path: "/" }
route:
cluster: app-service
# special case for when API requests are made, can just forward, not to minio
- match: { prefix: "/api/" }
route:
cluster: app-service
timeout: 120s
- match: { prefix: "/worker/" }
route:
cluster: worker-service
prefix_rewrite: "/"
- match: { prefix: "/db/" }
route:
cluster: couchdb-service
prefix_rewrite: "/"
# minio is on the default route because this works
# best, minio + AWS SDK doesn't handle path proxy
- match: { prefix: "/" }
route:
cluster: minio-service
http_filters:
- name: envoy.filters.http.router
clusters:
- name: app-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: app-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: app-service
port_value: 4002
- name: minio-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: minio-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: minio-service
port_value: 9000
- name: worker-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: worker-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: worker-service
port_value: 4003
- name: couchdb-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: couchdb-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: couchdb-service
port_value: 5984
- name: watchtower-service
connect_timeout: 0.25s
type: strict_dns
lb_policy: round_robin
load_assignment:
cluster_name: watchtower-service
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: watchtower-service
port_value: 8080

3
hosting/hosting.properties
View File

@ -18,7 +18,6 @@ WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
# An admin user can be automatically created initially if these are set
@ -26,4 +25,4 @@ BB_ADMIN_USER_EMAIL=
BB_ADMIN_USER_PASSWORD=
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
PLUGINS_DIR=
PLUGINS_DIR=

5
hosting/portainer/template.json
View File

@ -78,11 +78,6 @@
"default": "6379",
"preset": true
},
{
"name": "WATCHTOWER_PORT",
"default": "6161",
"preset": true
},
{
"name": "BUDIBASE_ENVIRONMENT",
"default": "PRODUCTION",

1
hosting/proxy/Dockerfile
View File

@ -22,5 +22,4 @@ ENV APPS_UPSTREAM_URL=http://app-service:4002
ENV WORKER_UPSTREAM_URL=http://worker-service:4003
ENV MINIO_UPSTREAM_URL=http://minio-service:9000
ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
ENV RESOLVER=127.0.0.11

5
hosting/proxy/nginx.prod.conf
View File

@ -81,7 +81,6 @@ http {
set $worker ${WORKER_UPSTREAM_URL};
set $minio ${MINIO_UPSTREAM_URL};
set $couchdb ${COUCHDB_UPSTREAM_URL};
set $watchtower ${WATCHTOWER_UPSTREAM_URL};
location /health {
access_log off;
@ -107,10 +106,6 @@ http {
proxy_pass $apps;
}
location = /v1/update {
proxy_pass $watchtower;
}
location ~ ^/(builder|app_) {
proxy_http_version 1.1;

3
hosting/scripts/airgapped/airgappedDockerBuild.js
View File

@ -12,7 +12,6 @@ let IMAGES = {
couch: "ibmcom/couchdb3",
curl: "curlimages/curl",
redis: "redis",
watchtower: "containrrr/watchtower",
}
if (IS_SINGLE_IMAGE) {
@ -53,4 +52,4 @@ if (!IS_SINGLE_IMAGE) {
copyFile(FILES.ENV)
// compress
execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`)
execSync(`tar -czf bb-airgapped.tar.gz hosting/scripts/bb-airgapped`)

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.1.1",
"version": "3.2.0",
"npmClient": "yarn",
"packages": [
"packages/*",

4
packages/backend-core/src/features/features.ts
View File

@ -271,8 +271,8 @@ export const flags = new FlagSet({
[FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
[FeatureFlag.SQS]: Flag.boolean(true),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
[FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
})
type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

52
packages/builder/src/pages/builder/portal/settings/version.svelte
View File

@ -4,12 +4,10 @@
Layout,
Heading,
Body,
Button,
Divider,
notifications,
Label,
Modal,
ModalContent,
Link,
} from "@budibase/bbui"
import { API } from "api"
import { auth, admin } from "stores/portal"
@ -21,8 +19,6 @@
let githubVersion
let githubPublishedDate
let githubPublishedTime
let needsUpdate = true
let updateModal
// Only admins allowed here
$: {
@ -31,21 +27,6 @@
}
}
async function updateBudibase() {
try {
notifications.info("Updating budibase..")
await fetch("/v1/update", {
headers: {
Authorization: "Bearer budibase",
},
})
notifications.success("Your budibase installation is up to date.")
getVersion()
} catch (err) {
notifications.error(`Error installing budibase update ${err}`)
}
}
async function getVersion() {
try {
version = await API.getBudibaseVersion()
@ -69,13 +50,6 @@
githubPublishedDate = new Date(githubResponse.published_at)
githubPublishedTime = githubPublishedDate.toLocaleTimeString()
githubPublishedDate = githubPublishedDate.toLocaleDateString()
//Does Budibase need to be updated?
if (githubVersion === version) {
needsUpdate = false
} else {
needsUpdate = true
}
} catch (error) {
notifications.error("Error getting the latest Budibase version")
githubVersion = null
@ -115,23 +89,15 @@
>
</Layout>
<Divider />
<div>
<Button cta on:click={updateModal.show} disabled={!needsUpdate}
>Update Budibase</Button
<Layout noPadding gap="XS">
<Heading>Updating Budibase</Heading>
<Body
>To update your self-host installation, follow the docs found <Link
size="L"
href="https://docs.budibase.com/docs/updating-budibase">here.</Link
></Body
>
<Modal bind:this={updateModal}>
<ModalContent
title="Update Budibase"
confirmText="Update"
onConfirm={updateBudibase}
>
<span
>Are you sure you want to update your budibase installation to the
latest version?</span
>
</ModalContent>
</Modal>
</div>
</Layout>
{/if}
</Layout>
{/if}

87
packages/server/src/automations/tests/scenarios/looping.spec.ts
View File

@ -355,6 +355,93 @@ describe("Loop automations", () => {
expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length)
})
it("should run an automation with a loop and update row step using stepIds", async () => {
const table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
value: {
name: "value",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const rows = [
{ name: "Row 1", value: 1, tableId: table._id },
{ name: "Row 2", value: 2, tableId: table._id },
{ name: "Row 3", value: 3, tableId: table._id },
]
await config.api.row.bulkImport(table._id!, { rows })
const builder = createAutomationBuilder({
name: "Test Loop and Update Row",
})
const results = await builder
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
},
{ stepId: "abc123" }
)
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.abc123.rows }}",
})
.updateRow({
rowId: "{{ loop.currentItem._id }}",
row: {
name: "Updated {{ loop.currentItem.name }}",
value: "{{ loop.currentItem.value }}",
tableId: table._id,
},
meta: {},
})
.queryRows({
tableId: table._id!,
})
.run()
const expectedRows = [
{ name: "Updated Row 1", value: 1 },
{ name: "Updated Row 2", value: 2 },
{ name: "Updated Row 3", value: 3 },
]
expect(results.steps[1].outputs.items).toEqual(
expect.arrayContaining(
expectedRows.map(row =>
expect.objectContaining({
success: true,
row: expect.objectContaining(row),
})
)
)
)
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining(
expectedRows.map(row => expect.objectContaining(row))
)
)
expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length)
expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length)
})
it("should run an automation with a loop and delete row step", async () => {
const table = await config.createTable({
name: "TestTable",

2
packages/server/src/threads/automation.ts
View File

@ -385,7 +385,7 @@ class Orchestrator {
stepIdx: number,
pathIdx?: number
): Promise<number> {
await processObject(loopStep.inputs, this.context)
await processObject(loopStep.inputs, this.processContext(this.context))
const iterations = getLoopIterations(loopStep)
let stepToLoopIndex = stepIdx + 1
let pathStepIdx = (pathIdx || stepIdx) + 1

3
packages/shared-core/src/helpers/async.ts Normal file
View File

@ -0,0 +1,3 @@
export async function wait(ms: number) {
return new Promise(resolve => setTimeout(resolve, ms))
}

2
packages/shared-core/src/helpers/index.ts
View File

@ -1,5 +1,7 @@
export * from "./helpers"
export * from "./integrations"
export * from "./async"
export * from "./retry"
export * as cron from "./cron"
export * as schema from "./schema"
export * as views from "./views"

28
packages/shared-core/src/helpers/retry.ts Normal file
View File

@ -0,0 +1,28 @@
import { wait } from "./async"
interface RetryOpts {
times?: number
}
export async function retry<T>(
fn: () => Promise<T>,
opts?: RetryOpts
): Promise<T> {
const { times = 3 } = opts || {}
if (times < 1) {
throw new Error(`invalid retry count: ${times}`)
}
let lastError: any
for (let i = 0; i < times; i++) {
const backoff = 1.5 ** i * 1000 * (Math.random() + 0.5)
await wait(backoff)
try {
return await fn()
} catch (e) {
lastError = e
}
}
throw lastError
}

16
packages/worker/src/api/controllers/system/environment.ts
View File

@ -2,6 +2,7 @@ import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
import { helpers } from "@budibase/shared-core"
let sqsAvailable: boolean
async function isSqsAvailable() {
@ -12,17 +13,22 @@ async function isSqsAvailable() {
}
try {
const couchInfo = dbCore.getCouchInfo()
if (!couchInfo.sqlUrl) {
const { url } = dbCore.getCouchInfo()
if (!url) {
sqsAvailable = false
return false
}
await nodeFetch(couchInfo.sqlUrl, {
timeout: 1000,
})
await helpers.retry(
async () => {
await nodeFetch(url, { timeout: 2000 })
},
{ times: 3 }
)
console.log("connected to SQS")
sqsAvailable = true
return true
} catch (e) {
console.warn("failed to connect to SQS", e)
sqsAvailable = false
return false
}