MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/rm-ff-code-per-user-per-creator

This commit is contained in:
José Vte. Calderón 2023-12-14 09:08:35 +01:00 committed by GitHub
parent 54ab1e16fe 99eb6597fb
commit ed3ad7dea9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 632 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
charts/budibase/README.md
View File

@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
| services.apps.replicaCount | int | `1` | The number of apps replicas to run. | | services.apps.replicaCount | int | `1` | The number of apps replicas to run. |
| services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. | | services.apps.resources | object | `{}` | The resources to use for apps pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
| services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> | | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. |
| services.automationWorkers.autoscaling.maxReplicas | int | `10` | |
| services.automationWorkers.autoscaling.minReplicas | int | `1` | |
| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. |
| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. |
| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. |
| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
| services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. | | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. |
| services.couchdb.backup.interval | string | `""` | Backup interval in seconds | | services.couchdb.backup.interval | string | `""` | Backup interval in seconds |
| services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. | | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |

9
charts/budibase/templates/app-service-deployment.yaml
View File

@ -192,7 +192,14 @@ spec:
- name: NODE_TLS_REJECT_UNAUTHORIZED - name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }} value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }} {{ end }}
{{- if .Values.services.automationWorkers.enabled }}
- name: APP_FEATURES
value: "api"
{{- end }}
{{- range .Values.services.apps.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always imagePullPolicy: Always
{{- if .Values.services.apps.startupProbe }} {{- if .Values.services.apps.startupProbe }}

248
charts/budibase/templates/automation-worker-service-deployment.yaml Normal file
View File

@ -0,0 +1,248 @@
{{- if .Values.services.automationWorkers.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
{{ if .Values.services.automationWorkers.deploymentAnnotations }}
{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}}
{{ end }}
labels:
io.kompose.service: automation-worker-service
{{ if .Values.services.automationWorkers.deploymentLabels }}
{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}}
{{ end }}
name: automation-worker-service
spec:
replicas: {{ .Values.services.automationWorkers.replicaCount }}
selector:
matchLabels:
io.kompose.service: automation-worker-service
strategy:
type: RollingUpdate
template:
metadata:
annotations:
{{ if .Values.services.automationWorkers.templateAnnotations }}
{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}}
{{ end }}
labels:
io.kompose.service: automation-worker-service
{{ if .Values.services.automationWorkers.templateLabels }}
{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}}
{{ end }}
spec:
containers:
- env:
- name: BUDIBASE_ENVIRONMENT
value: {{ .Values.globals.budibaseEnv }}
- name: DEPLOYMENT_ENVIRONMENT
value: "kubernetes"
- name: COUCH_DB_URL
{{ if .Values.services.couchdb.url }}
value: {{ .Values.services.couchdb.url }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:
secretKeyRef:
name: {{ template "couchdb.fullname" . }}
key: adminUsername
- name: COUCH_DB_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "couchdb.fullname" . }}
key: adminPassword
{{ end }}
- name: ENABLE_ANALYTICS
value: {{ .Values.globals.enableAnalytics | quote }}
- name: API_ENCRYPTION_KEY
value: {{ .Values.globals.apiEncryptionKey | quote }}
- name: HTTP_LOGGING
value: {{ .Values.services.automationWorkers.httpLogging | quote }}
- name: INTERNAL_API_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: internalApiKey
- name: INTERNAL_API_KEY_FALLBACK
value: {{ .Values.globals.internalApiKeyFallback | quote }}
- name: JWT_SECRET
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: jwtSecret
- name: JWT_SECRET_FALLBACK
value: {{ .Values.globals.jwtSecretFallback | quote }}
{{ if .Values.services.objectStore.region }}
- name: AWS_REGION
value: {{ .Values.services.objectStore.region }}
{{ end }}
- name: MINIO_ENABLED
value: {{ .Values.services.objectStore.minio | quote }}
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: objectStoreAccess
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: objectStoreSecret
- name: CLOUDFRONT_CDN
value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
- name: CLOUDFRONT_PUBLIC_KEY_ID
value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
- name: CLOUDFRONT_PRIVATE_KEY_64
value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
- name: PLUGIN_BUCKET_NAME
value: {{ .Values.services.objectStore.pluginBucketName | quote }}
- name: APPS_BUCKET_NAME
value: {{ .Values.services.objectStore.appsBucketName | quote }}
- name: GLOBAL_BUCKET_NAME
value: {{ .Values.services.objectStore.globalBucketName | quote }}
- name: BACKUPS_BUCKET_NAME
value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.automationWorkers.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}
- name: API_REQ_LIMIT_PER_SEC
value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }}
{{ end }}
- name: MULTI_TENANCY
value: {{ .Values.globals.multiTenancy | quote }}
- name: OFFLINE_MODE
value: {{ .Values.globals.offlineMode | quote }}
- name: LOG_LEVEL
value: {{ .Values.services.automationWorkers.logLevel | quote }}
- name: REDIS_PASSWORD
value: {{ .Values.services.redis.password }}
- name: REDIS_URL
{{ if .Values.services.redis.url }}
value: {{ .Values.services.redis.url }}
{{ else }}
value: redis-service:{{ .Values.services.redis.port }}
{{ end }}
- name: SELF_HOSTED
value: {{ .Values.globals.selfHosted | quote }}
- name: POSTHOG_TOKEN
value: {{ .Values.globals.posthogToken | quote }}
- name: WORKER_URL
value: http://worker-service:{{ .Values.services.worker.port }}
- name: PLATFORM_URL
value: {{ .Values.globals.platformUrl | quote }}
- name: ACCOUNT_PORTAL_URL
value: {{ .Values.globals.accountPortalUrl | quote }}
- name: ACCOUNT_PORTAL_API_KEY
value: {{ .Values.globals.accountPortalApiKey | quote }}
- name: COOKIE_DOMAIN
value: {{ .Values.globals.cookieDomain | quote }}
- name: HTTP_MIGRATIONS
value: {{ .Values.globals.httpMigrations | quote }}
- name: GOOGLE_CLIENT_ID
value: {{ .Values.globals.google.clientId | quote }}
- name: GOOGLE_CLIENT_SECRET
value: {{ .Values.globals.google.secret | quote }}
- name: AUTOMATION_MAX_ITERATIONS
value: {{ .Values.globals.automationMaxIterations | quote }}
- name: TENANT_FEATURE_FLAGS
value: {{ .Values.globals.tenantFeatureFlags | quote }}
- name: ENCRYPTION_KEY
value: {{ .Values.globals.bbEncryptionKey | quote }}
{{ if .Values.globals.bbAdminUserEmail }}
- name: BB_ADMIN_USER_EMAIL
value: {{ .Values.globals.bbAdminUserEmail | quote }}
{{ end }}
{{ if .Values.globals.bbAdminUserPassword }}
- name: BB_ADMIN_USER_PASSWORD
value: {{ .Values.globals.bbAdminUserPassword | quote }}
{{ end }}
{{ if .Values.globals.pluginsDir }}
- name: PLUGINS_DIR
value: {{ .Values.globals.pluginsDir | quote }}
{{ end }}
{{ if .Values.services.automationWorkers.nodeDebug }}
- name: NODE_DEBUG
value: {{ .Values.services.automationWorkers.nodeDebug | quote }}
{{ end }}
{{ if .Values.globals.datadogApmEnabled }}
- name: DD_LOGS_INJECTION
value: {{ .Values.globals.datadogApmEnabled | quote }}
- name: DD_APM_ENABLED
value: {{ .Values.globals.datadogApmEnabled | quote }}
- name: DD_APM_DD_URL
value: https://trace.agent.datadoghq.eu
{{ end }}
{{ if .Values.globals.globalAgentHttpProxy }}
- name: GLOBAL_AGENT_HTTP_PROXY
value: {{ .Values.globals.globalAgentHttpProxy | quote }}
{{ end }}
{{ if .Values.globals.globalAgentHttpsProxy }}
- name: GLOBAL_AGENT_HTTPS_PROXY
value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
{{ end }}
{{ if .Values.globals.globalAgentNoProxy }}
- name: GLOBAL_AGENT_NO_PROXY
value: {{ .Values.globals.globalAgentNoProxy | quote }}
{{ end }}
{{ if .Values.services.tlsRejectUnauthorized }}
- name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }}
- name: APP_FEATURES
value: "automations"
{{- range .Values.services.automationWorkers.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always
{{- if .Values.services.automationWorkers.startupProbe }}
{{- with .Values.services.automationWorkers.startupProbe }}
startupProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
{{- if .Values.services.automationWorkers.livenessProbe }}
{{- with .Values.services.automationWorkers.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
{{- if .Values.services.automationWorkers.readinessProbe }}
{{- with .Values.services.automationWorkers.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
name: bbautomationworker
ports:
- containerPort: {{ .Values.services.automationWorkers.port }}
{{ with .Values.services.automationWorkers.resources }}
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{ if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName | quote }}
{{ end }}
{{ if .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml .Values.imagePullSecrets | nindent 6 }}
{{ end }}
restartPolicy: Always
serviceAccountName: ""
status: {}
{{- end }}

32
charts/budibase/templates/automation-worker-service-hpa.yaml Normal file
View File

@ -0,0 +1,32 @@
{{- if .Values.services.automationWorkers.autoscaling.enabled }}
apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "budibase.fullname" . }}-apps
labels:
{{- include "budibase.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: automation-worker-service
minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }}
maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }}
metrics:
{{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- end }}

4
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -182,6 +182,10 @@ spec:
- name: NODE_TLS_REJECT_UNAUTHORIZED - name: NODE_TLS_REJECT_UNAUTHORIZED
value: {{ .Values.services.tlsRejectUnauthorized }} value: {{ .Values.services.tlsRejectUnauthorized }}
{{ end }} {{ end }}
{{- range .Values.services.worker.extraEnv }}
- name: {{ .name }}
value: {{ .value | quote }}
{{- end }}
image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
imagePullPolicy: Always imagePullPolicy: Always
{{- if .Values.services.worker.startupProbe }} {{- if .Values.services.worker.startupProbe }}

78
charts/budibase/values.yaml
View File

@ -220,6 +220,9 @@ services:
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these. # for more information on how to set these.
resources: {} resources: {}
# -- Extra environment variables to set for apps pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for apps pods. You shouldn't need to # -- Startup probe configuration for apps pods. You shouldn't need to
# change this, but if you want to you can find more information here: # change this, but if you want to you can find more information here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
@ -272,6 +275,78 @@ services:
# and resources set for the apps pods. # and resources set for the apps pods.
targetCPUUtilizationPercentage: 80 targetCPUUtilizationPercentage: 80
automationWorkers:
# -- Whether or not to enable the automation worker service. If you disable this,
# automations will be processed by the apps service.
enabled: true
# @ignore (you shouldn't need to change this)
port: 4002
# -- The number of automation worker replicas to run.
replicaCount: 1
# -- The log level for the automation worker service.
logLevel: info
# -- The resources to use for automation worker pods. See
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these.
resources: {}
# -- Extra environment variables to set for automation worker pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
startupProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
failureThreshold: 30
# @ignore
periodSeconds: 3
# -- Readiness probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
readinessProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
periodSeconds: 3
# @ignore
failureThreshold: 1
# -- Liveness probe configuration for automation worker pods. You shouldn't
# need to change this, but if you want to you can find more information
# here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>
# @default -- HTTP health checks.
livenessProbe:
# @ignore
httpGet:
path: /health
port: 4002
scheme: HTTP
# @ignore
failureThreshold: 3
# @ignore
periodSeconds: 30
autoscaling:
# -- Whether to enable horizontal pod autoscaling for the apps service.
enabled: false
minReplicas: 1
maxReplicas: 10
# -- Target CPU utilization percentage for the automation worker service.
# Note that for autoscaling to work, you will need to have metrics-server
# configured, and resources set for the automation worker pods.
targetCPUUtilizationPercentage: 80
worker: worker:
# @ignore (you shouldn't need to change this) # @ignore (you shouldn't need to change this)
port: 4003 port: 4003
@ -285,6 +360,9 @@ services:
# <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> # <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/>
# for more information on how to set these. # for more information on how to set these.
resources: {} resources: {}
# -- Extra environment variables to set for worker pods. Takes a list of
# name=value pairs.
extraEnv: []
# -- Startup probe configuration for worker pods. You shouldn't need to # -- Startup probe configuration for worker pods. You shouldn't need to
# change this, but if you want to you can find more information here: # change this, but if you want to you can find more information here:
# <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> # <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/>

2
lerna.json
View File

@ -1,5 +1,5 @@
{ {
"version": "2.13.37", "version": "2.13.39",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*", "packages/*",

13
packages/backend-core/src/cache/tests/writethrough.spec.ts vendored
View File

@ -1,15 +1,16 @@
import { DBTestConfiguration } from "../../../tests/extra" import { DBTestConfiguration } from "../../../tests/extra"
import { import { structures } from "../../../tests"
structures,
expectFunctionWasCalledTimesWith,
mocks,
} from "../../../tests"
import { Writethrough } from "../writethrough" import { Writethrough } from "../writethrough"
import { getDB } from "../../db" import { getDB } from "../../db"
import { Document } from "@budibase/types"
import tk from "timekeeper" import tk from "timekeeper"
tk.freeze(Date.now()) tk.freeze(Date.now())
interface ValueDoc extends Document {
value: any
}
const DELAY = 5000 const DELAY = 5000
describe("writethrough", () => { describe("writethrough", () => {
@ -117,7 +118,7 @@ describe("writethrough", () => {
describe("get", () => { describe("get", () => {
it("should be able to retrieve", async () => { it("should be able to retrieve", async () => {
await config.doInTenant(async () => { await config.doInTenant(async () => {
const response = await writethrough.get(docId) const response = await writethrough.get<ValueDoc>(docId)
expect(response.value).toBe(4) expect(response.value).toBe(4)
}) })
}) })

19
packages/backend-core/src/cache/writethrough.ts vendored
View File

@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl"
const DEFAULT_WRITE_RATE_MS = 10000 const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null let CACHE: BaseCache | null = null
interface CacheItem { interface CacheItem<T extends Document> {
doc: any doc: any
lastWrite: number lastWrite: number
} }
@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) {
return db.name + key return db.name + key
} }
function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { function makeCacheItem<T extends Document>(
doc: T,
lastWrite: number | null = null
): CacheItem<T> {
return { doc, lastWrite: lastWrite || Date.now() } return { doc, lastWrite: lastWrite || Date.now() }
} }
@ -35,7 +38,7 @@ async function put(
) { ) {
const cache = await getCache() const cache = await getCache()
const key = doc._id const key = doc._id
let cacheItem: CacheItem | undefined let cacheItem: CacheItem<any> | undefined
if (key) { if (key) {
cacheItem = await cache.get(makeCacheKey(db, key)) cacheItem = await cache.get(makeCacheKey(db, key))
} }
@ -84,12 +87,12 @@ async function put(
return { ok: true, id: output._id, rev: output._rev } return { ok: true, id: output._id, rev: output._rev }
} }
async function get(db: Database, id: string): Promise<any> { async function get<T extends Document>(db: Database, id: string): Promise<T> {
const cache = await getCache() const cache = await getCache()
const cacheKey = makeCacheKey(db, id) const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey) let cacheItem: CacheItem<T> = await cache.get(cacheKey)
if (!cacheItem) { if (!cacheItem) {
const doc = await db.get(id) const doc = await db.get<T>(id)
cacheItem = makeCacheItem(doc) cacheItem = makeCacheItem(doc)
await cache.store(cacheKey, cacheItem) await cache.store(cacheKey, cacheItem)
} }
@ -123,8 +126,8 @@ export class Writethrough {
return put(this.db, doc, writeRateMs) return put(this.db, doc, writeRateMs)
} }
async get(id: string) { async get<T extends Document>(id: string) {
return get(this.db, id) return get<T>(this.db, id)
} }
async remove(docOrId: any, rev?: any) { async remove(docOrId: any, rev?: any) {

1
packages/backend-core/src/environment.ts
View File

@ -107,6 +107,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(), API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

4
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -68,6 +68,10 @@ class InMemoryQueue {
}) })
} }
async isReady() {
return true
}
// simply puts a message to the queue and emits to the queue for processing // simply puts a message to the queue and emits to the queue for processing
/** /**
* Simple function to replicate the add message functionality of Bull, putting * Simple function to replicate the add message functionality of Bull, putting

1
packages/backend-core/src/redis/redlockImpl.ts
View File

@ -137,7 +137,6 @@ export async function doWithLock<T>(
const result = await task() const result = await task()
return { executed: true, result } return { executed: true, result }
} catch (e: any) { } catch (e: any) {
logWarn(`lock type: ${opts.type} error`, e)
// lock limit exceeded // lock limit exceeded
if (e.name === "LockError") { if (e.name === "LockError") {
if (opts.type === LockType.TRY_ONCE) { if (opts.type === LockType.TRY_ONCE) {

2
packages/pro

@ -1 +1 @@
Subproject commit 056c2093dbc93d9a10ea9f5050c84a84edd8100c Subproject commit 992486c10044a7495496b97bdf5f454d4020bfba

75
packages/server/scripts/dev/manage.js
View File

@ -1,7 +1,8 @@
#!/usr/bin/env node #!/usr/bin/env node
const compose = require("docker-compose") const compose = require("docker-compose")
const path = require("path") const path = require("path")
const fs = require("fs") const { parsed: existingConfig } = require("dotenv").config()
const updateDotEnv = require("update-dotenv")
// This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands. // This script wraps docker-compose allowing you to manage your dev infrastructure with simple commands.
const CONFIG = { const CONFIG = {
@ -17,45 +18,41 @@ const Commands = {
} }
async function init() { async function init() {
const envFilePath = path.join(process.cwd(), ".env") let config = {
if (!fs.existsSync(envFilePath)) { PORT: "4001",
const envFileJson = { MINIO_URL: "http://localhost:4004",
PORT: 4001, COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
MINIO_URL: "http://localhost:4004", REDIS_URL: "localhost:6379",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005", WORKER_URL: "http://localhost:4002",
REDIS_URL: "localhost:6379", INTERNAL_API_KEY: "budibase",
WORKER_URL: "http://localhost:4002", ACCOUNT_PORTAL_URL: "http://localhost:10001",
INTERNAL_API_KEY: "budibase", ACCOUNT_PORTAL_API_KEY: "budibase",
ACCOUNT_PORTAL_URL: "http://localhost:10001", PLATFORM_URL: "http://localhost:10000",
ACCOUNT_PORTAL_API_KEY: "budibase", JWT_SECRET: "testsecret",
PLATFORM_URL: "http://localhost:10000", ENCRYPTION_KEY: "testsecret",
JWT_SECRET: "testsecret", REDIS_PASSWORD: "budibase",
ENCRYPTION_KEY: "testsecret", MINIO_ACCESS_KEY: "budibase",
REDIS_PASSWORD: "budibase", MINIO_SECRET_KEY: "budibase",
MINIO_ACCESS_KEY: "budibase", COUCH_DB_PASSWORD: "budibase",
MINIO_SECRET_KEY: "budibase", COUCH_DB_USER: "budibase",
COUCH_DB_PASSWORD: "budibase", SELF_HOSTED: "1",
COUCH_DB_USER: "budibase", DISABLE_ACCOUNT_PORTAL: "1",
SELF_HOSTED: 1, MULTI_TENANCY: "",
DISABLE_ACCOUNT_PORTAL: 1, DISABLE_THREADING: "1",
MULTI_TENANCY: "", SERVICE: "app-service",
DISABLE_THREADING: 1, DEPLOYMENT_ENVIRONMENT: "development",
SERVICE: "app-service", BB_ADMIN_USER_EMAIL: "",
DEPLOYMENT_ENVIRONMENT: "development", BB_ADMIN_USER_PASSWORD: "",
BB_ADMIN_USER_EMAIL: "", PLUGINS_DIR: "",
BB_ADMIN_USER_PASSWORD: "", TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
PLUGINS_DIR: "", HTTP_MIGRATIONS: "0",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", HTTP_LOGGING: "0",
HTTP_MIGRATIONS: "0", VERSION: "0.0.0+local",
HTTP_LOGGING: "0",
VERSION: "0.0.0+local",
}
let envFile = ""
Object.keys(envFileJson).forEach(key => {
envFile += `${key}=${envFileJson[key]}\n`
})
fs.writeFileSync(envFilePath, envFile)
} }
config = { ...config, ...existingConfig }
await updateDotEnv(config)
} }
async function up() { async function up() {

105
packages/server/src/api/index.ts
View File

@ -4,62 +4,75 @@ import currentApp from "../middleware/currentapp"
import zlib from "zlib" import zlib from "zlib"
import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
import { middleware as pro } from "@budibase/pro" import { middleware as pro } from "@budibase/pro"
import { apiEnabled, automationsEnabled } from "../features"
import migrations from "../middleware/appMigrations" import migrations from "../middleware/appMigrations"
import { automationQueue } from "../automations"
export { shutdown } from "./routes/public" export { shutdown } from "./routes/public"
const compress = require("koa-compress") const compress = require("koa-compress")
export const router: Router = new Router() export const router: Router = new Router()
router.get("/health", ctx => (ctx.status = 200)) router.get("/health", async ctx => {
if (automationsEnabled()) {
if (!(await automationQueue.isReady())) {
ctx.status = 503
return
}
}
ctx.status = 200
})
router.get("/version", ctx => (ctx.body = envCore.VERSION)) router.get("/version", ctx => (ctx.body = envCore.VERSION))
router.use(middleware.errorHandling) router.use(middleware.errorHandling)
router // only add the routes if they are enabled
.use( if (apiEnabled()) {
compress({ router
threshold: 2048, .use(
gzip: { compress({
flush: zlib.constants.Z_SYNC_FLUSH, threshold: 2048,
}, gzip: {
deflate: { flush: zlib.constants.Z_SYNC_FLUSH,
flush: zlib.constants.Z_SYNC_FLUSH, },
}, deflate: {
br: false, flush: zlib.constants.Z_SYNC_FLUSH,
}) },
) br: false,
// re-direct before any middlewares occur })
.redirect("/", "/builder") )
.use( // re-direct before any middlewares occur
auth.buildAuthMiddleware([], { .redirect("/", "/builder")
publicAllowed: true, .use(
}) auth.buildAuthMiddleware([], {
) publicAllowed: true,
// nothing in the server should allow query string tenants })
// the server can be public anywhere, so nowhere should throw errors )
// if the tenancy has not been set, it'll have to be discovered at application layer // nothing in the server should allow query string tenants
.use( // the server can be public anywhere, so nowhere should throw errors
auth.buildTenancyMiddleware([], [], { // if the tenancy has not been set, it'll have to be discovered at application layer
noTenancyRequired: true, .use(
}) auth.buildTenancyMiddleware([], [], {
) noTenancyRequired: true,
.use(pro.licensing()) })
// @ts-ignore )
.use(currentApp) .use(pro.licensing())
.use(auth.auditLog) // @ts-ignore
// @ts-ignore .use(currentApp)
.use(migrations) .use(auth.auditLog)
// @ts-ignore
.use(migrations)
// authenticated routes // authenticated routes
for (let route of mainRoutes) { for (let route of mainRoutes) {
router.use(route.routes()) router.use(route.routes())
router.use(route.allowedMethods()) router.use(route.allowedMethods())
}
router.use(publicRoutes.routes())
router.use(publicRoutes.allowedMethods())
// WARNING - static routes will catch everything else after them this must be last
router.use(staticRoutes.routes())
router.use(staticRoutes.allowedMethods())
} }
router.use(publicRoutes.routes())
router.use(publicRoutes.allowedMethods())
// WARNING - static routes will catch everything else after them this must be last
router.use(staticRoutes.routes())
router.use(staticRoutes.allowedMethods())

10
packages/server/src/app.ts
View File

@ -9,7 +9,6 @@ import { ServiceType } from "@budibase/types"
import { env as coreEnv } from "@budibase/backend-core" import { env as coreEnv } from "@budibase/backend-core"
coreEnv._set("SERVICE_TYPE", ServiceType.APPS) coreEnv._set("SERVICE_TYPE", ServiceType.APPS)
import { apiEnabled } from "./features"
import createKoaApp from "./koa" import createKoaApp from "./koa"
import Koa from "koa" import Koa from "koa"
import { Server } from "http" import { Server } from "http"
@ -18,12 +17,9 @@ import { startup } from "./startup"
let app: Koa, server: Server let app: Koa, server: Server
async function start() { async function start() {
// if API disabled, could run automations instead const koa = createKoaApp()
if (apiEnabled()) { app = koa.app
const koa = createKoaApp() server = koa.server
app = koa.app
server = koa.server
}
// startup includes automation runner - if enabled // startup includes automation runner - if enabled
await startup(app, server) await startup(app, server)
} }

7
packages/server/src/features.ts
View File

@ -22,3 +22,10 @@ export function automationsEnabled() {
export function apiEnabled() { export function apiEnabled() {
return featureList.includes(AppFeature.API) return featureList.includes(AppFeature.API)
} }
export function printFeatures() {
if (!env.APP_FEATURES) {
return
}
console.log(`**** APP FEATURES SET: ${featureList.join(", ")} ****`)
}

21
packages/server/src/startup.ts
View File

@ -19,11 +19,14 @@ import * as pro from "@budibase/pro"
import * as api from "./api" import * as api from "./api"
import sdk from "./sdk" import sdk from "./sdk"
import { initialise as initialiseWebsockets } from "./websockets" import { initialise as initialiseWebsockets } from "./websockets"
import { automationsEnabled } from "./features" import { automationsEnabled, printFeatures } from "./features"
import Koa from "koa"
import { Server } from "http"
import { AddressInfo } from "net"
let STARTUP_RAN = false let STARTUP_RAN = false
async function initRoutes(app: any) { async function initRoutes(app: Koa) {
if (!env.isTest()) { if (!env.isTest()) {
const plugin = await bullboard.init() const plugin = await bullboard.init()
app.use(plugin) app.use(plugin)
@ -48,27 +51,31 @@ async function initPro() {
}) })
} }
function shutdown(server?: any) { function shutdown(server?: Server) {
if (server) { if (server) {
server.close() server.close()
server.destroy() server.destroy()
} }
} }
export async function startup(app?: any, server?: any) { export async function startup(app?: Koa, server?: Server) {
if (STARTUP_RAN) { if (STARTUP_RAN) {
return return
} }
printFeatures()
STARTUP_RAN = true STARTUP_RAN = true
if (server && !env.CLUSTER_MODE) { if (app && server && !env.CLUSTER_MODE) {
console.log(`Budibase running on ${JSON.stringify(server.address())}`) console.log(`Budibase running on ${JSON.stringify(server.address())}`)
env._set("PORT", server.address().port) const address = server.address() as AddressInfo
env._set("PORT", address.port)
} }
eventEmitter.emitPort(env.PORT) eventEmitter.emitPort(env.PORT)
fileSystem.init() fileSystem.init()
await redis.init() await redis.init()
eventInit() eventInit()
initialiseWebsockets(app, server) if (app && server) {
initialiseWebsockets(app, server)
}
// run migrations on startup if not done via http // run migrations on startup if not done via http
// not recommended in a clustered environment // not recommended in a clustered environment

1
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -17,7 +17,6 @@ import {
basicWebhook, basicWebhook,
} from "./structures" } from "./structures"
import { import {
auth,
cache, cache,
constants, constants,
context, context,

70
packages/worker/scripts/dev/manage.js
View File

@ -1,44 +1,40 @@
#!/usr/bin/env node #!/usr/bin/env node
const path = require("path") const { parsed: existingConfig } = require("dotenv").config()
const fs = require("fs") const updateDotEnv = require("update-dotenv")
async function init() { async function init() {
const envFilePath = path.join(process.cwd(), ".env") let config = {
if (!fs.existsSync(envFilePath)) { SELF_HOSTED: "1",
const envFileJson = { PORT: "4002",
SELF_HOSTED: 1, CLUSTER_PORT: "10000",
PORT: 4002, JWT_SECRET: "testsecret",
CLUSTER_PORT: 10000, INTERNAL_API_KEY: "budibase",
JWT_SECRET: "testsecret", MINIO_ACCESS_KEY: "budibase",
INTERNAL_API_KEY: "budibase", MINIO_SECRET_KEY: "budibase",
MINIO_ACCESS_KEY: "budibase", REDIS_URL: "localhost:6379",
MINIO_SECRET_KEY: "budibase", REDIS_PASSWORD: "budibase",
REDIS_URL: "localhost:6379", MINIO_URL: "http://localhost:4004",
REDIS_PASSWORD: "budibase", COUCH_DB_URL: "http://budibase:budibase@localhost:4005",
MINIO_URL: "http://localhost:4004", COUCH_DB_USERNAME: "budibase",
COUCH_DB_URL: "http://budibase:budibase@localhost:4005", COUCH_DB_PASSWORD: "budibase",
COUCH_DB_USERNAME: "budibase", // empty string is false
COUCH_DB_PASSWORD: "budibase", MULTI_TENANCY: "",
// empty string is false DISABLE_ACCOUNT_PORTAL: "1",
MULTI_TENANCY: "", ACCOUNT_PORTAL_URL: "http://localhost:10001",
DISABLE_ACCOUNT_PORTAL: 1, ACCOUNT_PORTAL_API_KEY: "budibase",
ACCOUNT_PORTAL_URL: "http://localhost:10001", PLATFORM_URL: "http://localhost:10000",
ACCOUNT_PORTAL_API_KEY: "budibase", APPS_URL: "http://localhost:4001",
PLATFORM_URL: "http://localhost:10000", SERVICE: "worker-service",
APPS_URL: "http://localhost:4001", DEPLOYMENT_ENVIRONMENT: "development",
SERVICE: "worker-service", TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR",
DEPLOYMENT_ENVIRONMENT: "development", ENABLE_EMAIL_TEST_MODE: "1",
TENANT_FEATURE_FLAGS: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR", HTTP_LOGGING: "0",
ENABLE_EMAIL_TEST_MODE: 1, VERSION: "0.0.0+local",
HTTP_LOGGING: 0,
VERSION: "0.0.0+local",
}
let envFile = ""
Object.keys(envFileJson).forEach(key => {
envFile += `${key}=${envFileJson[key]}\n`
})
fs.writeFileSync(envFilePath, envFile)
} }
config = { ...config, ...existingConfig }
await updateDotEnv(config)
} }
// if more than init required use this to determine the command type // if more than init required use this to determine the command type

25
packages/worker/src/api/controllers/system/environment.ts
View File

@ -1,6 +1,25 @@
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import env from "../../../environment" import env from "../../../environment"
import { env as coreEnv } from "@budibase/backend-core" import { env as coreEnv } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
let sqsAvailable: boolean
async function isSqsAvailable() {
if (sqsAvailable !== undefined) {
return sqsAvailable
}
try {
await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
timeout: 1000,
})
sqsAvailable = true
return true
} catch (e) {
sqsAvailable = false
return false
}
}
export const fetch = async (ctx: Ctx) => { export const fetch = async (ctx: Ctx) => {
ctx.body = { ctx.body = {
@ -12,4 +31,10 @@ export const fetch = async (ctx: Ctx) => {
baseUrl: env.PLATFORM_URL, baseUrl: env.PLATFORM_URL,
isDev: env.isDev() && !env.isTest(), isDev: env.isDev() && !env.isTest(),
} }
if (env.SELF_HOSTED) {
ctx.body.infrastructure = {
sqs: await isSqsAvailable(),
}
}
} }

19
packages/worker/src/api/routes/system/tests/environment.spec.ts
View File

@ -1,5 +1,7 @@
import { TestConfiguration } from "../../../../tests" import { TestConfiguration } from "../../../../tests"
jest.unmock("node-fetch")
describe("/api/system/environment", () => { describe("/api/system/environment", () => {
const config = new TestConfiguration() const config = new TestConfiguration()
@ -27,5 +29,22 @@ describe("/api/system/environment", () => {
offlineMode: false, offlineMode: false,
}) })
}) })
it("returns the expected environment for self hosters", async () => {
await config.withEnv({ SELF_HOSTED: true }, async () => {
const env = await config.api.environment.getEnvironment()
expect(env.body).toEqual({
cloud: false,
disableAccountPortal: 0,
isDev: false,
multiTenancy: true,
baseUrl: "http://localhost:10000",
offlineMode: false,
infrastructure: {
sqs: false,
},
})
})
})
}) })
}) })

29
packages/worker/src/tests/TestConfiguration.ts
View File

@ -36,6 +36,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import API from "./api" import API from "./api"
import jwt, { Secret } from "jsonwebtoken" import jwt, { Secret } from "jsonwebtoken"
import cloneDeep from "lodash/fp/cloneDeep"
class TestConfiguration { class TestConfiguration {
server: any server: any
@ -240,6 +241,34 @@ class TestConfiguration {
return { message: "Admin user only endpoint.", status: 403 } return { message: "Admin user only endpoint.", status: 403 }
} }
async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
let cleanup = this.setEnv(newEnvVars)
try {
await f()
} finally {
cleanup()
}
}
/*
* Sets the environment variables to the given values and returns a function
* that can be called to reset the environment variables to their original values.
*/
setEnv(newEnvVars: Partial<typeof env>): () => void {
const oldEnv = cloneDeep(env)
let key: keyof typeof newEnvVars
for (key in newEnvVars) {
env._set(key, newEnvVars[key])
}
return () => {
for (const [key, value] of Object.entries(oldEnv)) {
env._set(key, value)
}
}
}
// USERS // USERS
async createDefaultUser() { async createDefaultUser() {