From 54ab1e16fee631eccf69f0c4c77a0e102cb7e86f Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Tue, 12 Dec 2023 17:40:27 +0100 Subject: [PATCH 01/65] Upgrade account-portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index a0b13270c3..b090705d6c 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit a0b13270c36dd188e2a953d026b4560a1208008e +Subproject commit b090705d6c17029c1d0a2ff991d045de20e8e52e From 6098a2d9e8b01a3f5214acb0b10b7c0e1bbac3a1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 10:47:16 +0000 Subject: [PATCH 02/65] Split out automation processing from the apps pods. --- charts/budibase/README.md | 10 + .../templates/app-service-deployment.yaml | 2 + .../automation-worker-service-deployment.yaml | 243 ++++++++++++++++++ .../automation-worker-service-hpa.yaml | 32 +++ .../automation-worker-service-service.yaml | 15 ++ charts/budibase/values.yaml | 66 +++++ packages/server/src/api/index.ts | 12 +- 7 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 charts/budibase/templates/automation-worker-service-deployment.yaml create mode 100644 charts/budibase/templates/automation-worker-service-hpa.yaml create mode 100644 charts/budibase/templates/automation-worker-service-service.yaml diff --git a/charts/budibase/README.md b/charts/budibase/README.md index d8191026ce..5fc1b606ad 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -157,6 +157,16 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.replicaCount | int | `1` | The number of apps replicas to run. | | services.apps.resources | object | `{}` | The resources to use for apps pods. See for more information on how to set these. | | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. | +| services.automationWorkers.autoscaling.maxReplicas | int | `10` | | +| services.automationWorkers.autoscaling.minReplicas | int | `1` | | +| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | +| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. | +| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See for more information on how to set these. | +| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. | | services.couchdb.backup.interval | string | `""` | Backup interval in seconds | | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See for more information on how to set these. | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 7358e474ca..11651ee8bc 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,6 +192,8 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + - name: APP_FEATURES + value: "api" image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml new file mode 100644 index 0000000000..440fcc4e2a --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -0,0 +1,243 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: +{{ if .Values.services.automationWorkers.deploymentAnnotations }} +{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.deploymentLabels }} +{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}} +{{ end }} + name: automation-worker-service +spec: + replicas: {{ .Values.services.automationWorkers.replicaCount }} + selector: + matchLabels: + io.kompose.service: automation-worker-service + strategy: + type: RollingUpdate + template: + metadata: + annotations: +{{ if .Values.services.automationWorkers.templateAnnotations }} +{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.templateLabels }} +{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}} +{{ end }} + spec: + containers: + - env: + - name: BUDIBASE_ENVIRONMENT + value: {{ .Values.globals.budibaseEnv }} + - name: DEPLOYMENT_ENVIRONMENT + value: "kubernetes" + - name: COUCH_DB_URL + {{ if .Values.services.couchdb.url }} + value: {{ .Values.services.couchdb.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} + {{ end }} + {{ if .Values.services.couchdb.enabled }} + - name: COUCH_DB_USER + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminUsername + - name: COUCH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminPassword + {{ end }} + - name: ENABLE_ANALYTICS + value: {{ .Values.globals.enableAnalytics | quote }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.automationWorkers.httpLogging | quote }} + - name: INTERNAL_API_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} + {{ if .Values.services.objectStore.region }} + - name: AWS_REGION + value: {{ .Values.services.objectStore.region }} + {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreAccess + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreSecret + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} + - name: MINIO_URL + value: {{ .Values.services.objectStore.url }} + - name: PLUGIN_BUCKET_NAME + value: {{ .Values.services.objectStore.pluginBucketName | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | quote }} + - name: GLOBAL_BUCKET_NAME + value: {{ .Values.services.objectStore.globalBucketName | quote }} + - name: BACKUPS_BUCKET_NAME + value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: PORT + value: {{ .Values.services.automationWorkers.port | quote }} + {{ if .Values.services.worker.publicApiRateLimitPerSecond }} + - name: API_REQ_LIMIT_PER_SEC + value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }} + {{ end }} + - name: MULTI_TENANCY + value: {{ .Values.globals.multiTenancy | quote }} + - name: OFFLINE_MODE + value: {{ .Values.globals.offlineMode | quote }} + - name: LOG_LEVEL + value: {{ .Values.services.automationWorkers.logLevel | quote }} + - name: REDIS_PASSWORD + value: {{ .Values.services.redis.password }} + - name: REDIS_URL + {{ if .Values.services.redis.url }} + value: {{ .Values.services.redis.url }} + {{ else }} + value: redis-service:{{ .Values.services.redis.port }} + {{ end }} + - name: SELF_HOSTED + value: {{ .Values.globals.selfHosted | quote }} + - name: POSTHOG_TOKEN + value: {{ .Values.globals.posthogToken | quote }} + - name: WORKER_URL + value: http://worker-service:{{ .Values.services.worker.port }} + - name: PLATFORM_URL + value: {{ .Values.globals.platformUrl | quote }} + - name: ACCOUNT_PORTAL_URL + value: {{ .Values.globals.accountPortalUrl | quote }} + - name: ACCOUNT_PORTAL_API_KEY + value: {{ .Values.globals.accountPortalApiKey | quote }} + - name: COOKIE_DOMAIN + value: {{ .Values.globals.cookieDomain | quote }} + - name: HTTP_MIGRATIONS + value: {{ .Values.globals.httpMigrations | quote }} + - name: GOOGLE_CLIENT_ID + value: {{ .Values.globals.google.clientId | quote }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.globals.google.secret | quote }} + - name: AUTOMATION_MAX_ITERATIONS + value: {{ .Values.globals.automationMaxIterations | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} + - name: ENCRYPTION_KEY + value: {{ .Values.globals.bbEncryptionKey | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: {{ .Values.globals.bbAdminUserEmail | quote }} + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: {{ .Values.globals.bbAdminUserPassword | quote }} + {{ end }} + {{ if .Values.globals.pluginsDir }} + - name: PLUGINS_DIR + value: {{ .Values.globals.pluginsDir | quote }} + {{ end }} + {{ if .Values.services.automationWorkers.nodeDebug }} + - name: NODE_DEBUG + value: {{ .Values.services.automationWorkers.nodeDebug | quote }} + {{ end }} + {{ if .Values.globals.datadogApmEnabled }} + - name: DD_LOGS_INJECTION + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_ENABLED + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_DD_URL + value: https://trace.agent.datadoghq.eu + {{ end }} + {{ if .Values.globals.globalAgentHttpProxy }} + - name: GLOBAL_AGENT_HTTP_PROXY + value: {{ .Values.globals.globalAgentHttpProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentHttpsProxy }} + - name: GLOBAL_AGENT_HTTPS_PROXY + value: {{ .Values.globals.globalAgentHttpsProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentNoProxy }} + - name: GLOBAL_AGENT_NO_PROXY + value: {{ .Values.globals.globalAgentNoProxy | quote }} + {{ end }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} + - name: APP_FEATURES + # We keep `api` enabled here for the /health endpoint + value: "automations,api" + + image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} + imagePullPolicy: Always + {{- if .Values.services.automationWorkers.startupProbe }} + {{- with .Values.services.automationWorkers.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.livenessProbe }} + {{- with .Values.services.automationWorkers.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.readinessProbe }} + {{- with .Values.services.automationWorkers.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + name: bbapps + ports: + - containerPort: {{ .Values.services.automationWorkers.port }} + {{ with .Values.services.automationWorkers.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{ end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName | quote }} + {{ end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} + restartPolicy: Always + serviceAccountName: "" +status: {} diff --git a/charts/budibase/templates/automation-worker-service-hpa.yaml b/charts/budibase/templates/automation-worker-service-hpa.yaml new file mode 100644 index 0000000000..f29223b61b --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.services.automationWorkers.autoscaling.enabled }} +apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "budibase.fullname" . }}-apps + labels: + {{- include "budibase.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: automation-worker-service + minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }} + maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }} + metrics: + {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/budibase/templates/automation-worker-service-service.yaml b/charts/budibase/templates/automation-worker-service-service.yaml new file mode 100644 index 0000000000..1b65227c60 --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + io.kompose.service: automation-worker-service + name: automation-worker-service +spec: + ports: + - name: {{ .Values.services.automationWorkers.port | quote }} + port: {{ .Values.services.automationWorkers.port }} + targetPort: {{ .Values.services.automationWorkers.port }} + selector: + io.kompose.service: automation-worker-service +status: + loadBalancer: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 13054e75fc..0fcc599d23 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -272,6 +272,72 @@ services: # and resources set for the apps pods. targetCPUUtilizationPercentage: 80 + automationWorkers: + # @ignore (you shouldn't need to change this) + port: 4002 + # -- The number of automation worker replicas to run. + replicaCount: 1 + # -- The log level for the automation worker service. + logLevel: info + # -- The resources to use for automation worker pods. See + # + # for more information on how to set these. + resources: {} + # -- Startup probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + startupProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 30 + # @ignore + periodSeconds: 3 + # -- Readiness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + readinessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + periodSeconds: 3 + # @ignore + failureThreshold: 1 + # -- Liveness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + livenessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 3 + # @ignore + periodSeconds: 30 + autoscaling: + # -- Whether to enable horizontal pod autoscaling for the apps service. + enabled: false + minReplicas: 1 + maxReplicas: 10 + # -- Target CPU utilization percentage for the automation worker service. + # Note that for autoscaling to work, you will need to have metrics-server + # configured, and resources set for the automation worker pods. + targetCPUUtilizationPercentage: 80 + worker: # @ignore (you shouldn't need to change this) port: 4003 diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index a01e3764f0..444a346e72 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -5,13 +5,23 @@ import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" import migrations from "../middleware/appMigrations" +import { automationsEnabled } from "src/features" +import { automationQueue } from "src/automations" export { shutdown } from "./routes/public" const compress = require("koa-compress") export const router: Router = new Router() -router.get("/health", ctx => (ctx.status = 200)) +router.get("/health", async ctx => { + if (automationsEnabled()) { + if (!(await automationQueue.isReady())) { + ctx.status = 503 + return + } + } + ctx.status = 200 +}) router.get("/version", ctx => (ctx.body = envCore.VERSION)) router.use(middleware.errorHandling) From ff04d0516ad9097ec335de965b2e9933e8f9f556 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 10:59:35 +0000 Subject: [PATCH 03/65] Disable API completely in automations workers. --- .../templates/automation-worker-service-deployment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 440fcc4e2a..83071f17ac 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -193,8 +193,7 @@ spec: value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - name: APP_FEATURES - # We keep `api` enabled here for the /health endpoint - value: "automations,api" + value: "automations" image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always From dbaa2699ca163e2a3da96bb8d5e4f9437953ad29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 11:10:58 +0000 Subject: [PATCH 04/65] Fix broken imports. --- packages/server/src/api/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index 444a346e72..9eba33126e 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -5,8 +5,8 @@ import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" import migrations from "../middleware/appMigrations" -import { automationsEnabled } from "src/features" -import { automationQueue } from "src/automations" +import { automationsEnabled } from "../features" +import { automationQueue } from "../automations" export { shutdown } from "./routes/public" const compress = require("koa-compress") From 4934b4a4282034fe140915e1825e6516f9cf54b5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 11:58:56 +0000 Subject: [PATCH 05/65] Fix broken test. --- packages/backend-core/src/queue/inMemoryQueue.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index a8add7ecb6..ac7cdf550b 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -68,6 +68,10 @@ class InMemoryQueue { }) } + async isReady() { + return true + } + // simply puts a message to the queue and emits to the queue for processing /** * Simple function to replicate the add message functionality of Bull, putting From 51c328996837a52fd04e311295208d9c0c334180 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:30:11 +0000 Subject: [PATCH 06/65] Fix lint error. --- packages/server/src/api/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts index f4a9875855..ad3d8307da 100644 --- a/packages/server/src/api/index.ts +++ b/packages/server/src/api/index.ts @@ -4,9 +4,8 @@ import currentApp from "../middleware/currentapp" import zlib from "zlib" import { mainRoutes, staticRoutes, publicRoutes } from "./routes" import { middleware as pro } from "@budibase/pro" -import { apiEnabled } from "../features" +import { apiEnabled, automationsEnabled } from "../features" import migrations from "../middleware/appMigrations" -import { automationsEnabled } from "../features" import { automationQueue } from "../automations" export { shutdown } from "./routes/public" From d164a3788db9f4b8a0f389c944e9bcc84c9ac456 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:36:24 +0000 Subject: [PATCH 07/65] Make it possible to disable automation workers. --- charts/budibase/README.md | 1 + .../templates/app-service-deployment.yaml | 3 +++ .../automation-worker-service-deployment.yaml | 2 ++ .../automation-worker-service-service.yaml | 15 --------------- charts/budibase/values.yaml | 3 +++ 5 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 charts/budibase/templates/automation-worker-service-service.yaml diff --git a/charts/budibase/README.md b/charts/budibase/README.md index 5fc1b606ad..342011bdb1 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -161,6 +161,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.automationWorkers.autoscaling.maxReplicas | int | `10` | | | services.automationWorkers.autoscaling.minReplicas | int | `1` | | | services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | | services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 11651ee8bc..99d278ef97 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,8 +192,11 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{ }} + {{- if .Values.services.automationWorkers.enabled }} - name: APP_FEATURES value: "api" + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 83071f17ac..a83542607a 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -1,3 +1,4 @@ +{{- if .Values.services.automationWorkers.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -240,3 +241,4 @@ spec: restartPolicy: Always serviceAccountName: "" status: {} +{{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/automation-worker-service-service.yaml b/charts/budibase/templates/automation-worker-service-service.yaml deleted file mode 100644 index 1b65227c60..0000000000 --- a/charts/budibase/templates/automation-worker-service-service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - io.kompose.service: automation-worker-service - name: automation-worker-service -spec: - ports: - - name: {{ .Values.services.automationWorkers.port | quote }} - port: {{ .Values.services.automationWorkers.port }} - targetPort: {{ .Values.services.automationWorkers.port }} - selector: - io.kompose.service: automation-worker-service -status: - loadBalancer: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 0fcc599d23..6dbcc20242 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -273,6 +273,9 @@ services: targetCPUUtilizationPercentage: 80 automationWorkers: + # -- Whether or not to enable the automation worker service. If you disable this, + # automations will be processed by the apps service. + enabled: true # @ignore (you shouldn't need to change this) port: 4002 # -- The number of automation worker replicas to run. From 33818daea9cce66a24d848416d9352ba2926ba94 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 12:37:08 +0000 Subject: [PATCH 08/65] Fix whitespace. --- charts/budibase/templates/app-service-deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 99d278ef97..495d11aadc 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,12 +192,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - {{ }} {{- if .Values.services.automationWorkers.enabled }} - name: APP_FEATURES value: "api" {{- end }} - image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} From 59581890cbbea5a319dd6101e5c015c9641cc841 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 15:48:22 +0000 Subject: [PATCH 09/65] Rename automation worker container. --- .../templates/automation-worker-service-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index a83542607a..4c41be3393 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -216,7 +216,7 @@ spec: {{- toYaml . | nindent 10 }} {{- end }} {{- end }} - name: bbapps + name: bbautomationworker ports: - containerPort: {{ .Values.services.automationWorkers.port }} {{ with .Values.services.automationWorkers.resources }} From 79fa567095c608b7e7a11a34624d9e60373450f8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 13 Dec 2023 17:10:19 +0000 Subject: [PATCH 10/65] Allow the passing of extra envrionment variables to apps, worker, and automation worker pods. --- charts/budibase/templates/app-service-deployment.yaml | 4 ++++ .../templates/automation-worker-service-deployment.yaml | 4 ++++ charts/budibase/templates/worker-service-deployment.yaml | 4 ++++ charts/budibase/values.yaml | 9 +++++++++ 4 files changed, 21 insertions(+) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 495d11aadc..9fb435c2a3 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -196,6 +196,10 @@ spec: - name: APP_FEATURES value: "api" {{- end }} + {{- range .Values.services.apps.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 4c41be3393..46be6a4435 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -195,6 +195,10 @@ spec: {{ end }} - name: APP_FEATURES value: "automations" + {{- range .Values.services.automationWorkers.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 6427aa70e8..1d90aaf954 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -182,6 +182,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{- range .Values.services.worker.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.worker.startupProbe }} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 6dbcc20242..09262df463 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -220,6 +220,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for apps pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for apps pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -286,6 +289,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for automation worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for automation worker pods. You shouldn't # need to change this, but if you want to you can find more information # here: @@ -354,6 +360,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for worker pods. You shouldn't need to # change this, but if you want to you can find more information here: # From 99eb6597fbc2562ebcca6ad14c5a90313049a9b7 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 13 Dec 2023 17:59:15 +0000 Subject: [PATCH 11/65] Bump version to 2.13.39 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 3d309f0881..ed49a4267d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.38", + "version": "2.13.39", "npmClient": "yarn", "packages": [ "packages/*", From 05b56925209eb4667276eb09c63dc9b7883bfe54 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 10:47:53 +0100 Subject: [PATCH 12/65] Add app migration test --- .../src/appMigrations/tests/migrations.spec.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 9d80cc5f99..9b1a0a9dd8 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,8 +1,21 @@ import { context } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" import { MIGRATIONS } from "../migrations" +import { getAppMigrationVersion } from "../appMigrationMetadata" +import { latestMigration } from ".." + +describe("migrations", () => { + it("new apps are created with the latest app migration version set", async () => { + const config = setup.getConfig() + await config.init() + + await config.doInContext(config.getAppId(), async () => { + const migrationVersion = await getAppMigrationVersion(config.getAppId()) + + expect(migrationVersion).toEqual(latestMigration) + }) + }) -describe("migration", () => { // These test is checking that each migration is "idempotent". // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran it("each migration can rerun safely", async () => { From 34d3edc2dbc467d55347fa53b912dfa587347b73 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 10:50:13 +0100 Subject: [PATCH 13/65] Add header --- packages/backend-core/src/constants/misc.ts | 1 + packages/server/src/appMigrations/index.ts | 13 ++++++++++++- packages/server/src/middleware/appMigrations.ts | 4 +--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 8ef34196ed..139bd7c0d3 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -28,6 +28,7 @@ export enum Header { CSRF_TOKEN = "x-csrf-token", CORRELATION_ID = "x-budibase-correlation-id", AUTHORIZATION = "authorization", + MIGRATING_APP = "x-budibase-migrating-app", } export enum GlobalRole { diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index a4ffe64604..9bc70f0abc 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -1,6 +1,9 @@ import queue from "./queue" +import { Next } from "koa" import { getAppMigrationVersion } from "./appMigrationMetadata" import { MIGRATIONS } from "./migrations" +import { UserCtx } from "@budibase/types" +import { Header } from "@budibase/backend-core" export * from "./appMigrationMetadata" @@ -15,7 +18,11 @@ export const latestMigration = MIGRATIONS.map(m => m.id) const getTimestamp = (versionId: string) => versionId?.split("_")[0] -export async function checkMissingMigrations(appId: string) { +export async function checkMissingMigrations( + ctx: UserCtx, + next: Next, + appId: string +) { const currentVersion = await getAppMigrationVersion(appId) if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { @@ -29,5 +36,9 @@ export async function checkMissingMigrations(appId: string) { removeOnFail: true, } ) + + ctx.response.set(Header.MIGRATING_APP, appId) } + + return next() } diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts index a94b8823e8..36e021c7ed 100644 --- a/packages/server/src/middleware/appMigrations.ts +++ b/packages/server/src/middleware/appMigrations.ts @@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => { return next() } - await checkMissingMigrations(appId) - - return next() + return checkMissingMigrations(ctx, next, appId) } From 9396c2fd6a5d5f67441fe889ed88e2940d0260ee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:05:30 +0100 Subject: [PATCH 14/65] Split tests --- .../tests/migrations.integrity.spec.ts | 25 ++++++++++++ .../appMigrations/tests/migrations.spec.ts | 38 +++++++------------ 2 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 packages/server/src/appMigrations/tests/migrations.integrity.spec.ts diff --git a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts new file mode 100644 index 0000000000..145a06d7f5 --- /dev/null +++ b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts @@ -0,0 +1,25 @@ +import { context } from "@budibase/backend-core" +import * as setup from "../../api/routes/tests/utilities" +import * as migrations from "../migrations" + +describe("migration integrity", () => { + // These test is checking that each migration is "idempotent". + // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran + it("each migration can rerun safely", async () => { + const config = setup.getConfig() + await config.init() + + await config.doInContext(config.getAppId(), async () => { + const db = context.getAppDB() + for (const migration of migrations.MIGRATIONS) { + await migration.func() + const docs = await db.allDocs({ include_docs: true }) + + await migration.func() + const latestDocs = await db.allDocs({ include_docs: true }) + + expect(docs).toEqual(latestDocs) + } + }) + }) +}) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 9b1a0a9dd8..d27a1ac37e 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,8 +1,18 @@ -import { context } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" -import { MIGRATIONS } from "../migrations" +import * as migrations from "../migrations" import { getAppMigrationVersion } from "../appMigrationMetadata" -import { latestMigration } from ".." +import { AppMigration } from ".." + +const mockedMigrations: AppMigration[] = [ + { + id: "20231211101320_test", + func: async () => {}, + }, +] + +jest.doMock("../migrations", () => ({ + MIGRATIONS: mockedMigrations, +})) describe("migrations", () => { it("new apps are created with the latest app migration version set", async () => { @@ -12,27 +22,7 @@ describe("migrations", () => { await config.doInContext(config.getAppId(), async () => { const migrationVersion = await getAppMigrationVersion(config.getAppId()) - expect(migrationVersion).toEqual(latestMigration) - }) - }) - - // These test is checking that each migration is "idempotent". - // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran - it("each migration can rerun safely", async () => { - const config = setup.getConfig() - await config.init() - - await config.doInContext(config.getAppId(), async () => { - const db = context.getAppDB() - for (const migration of MIGRATIONS) { - await migration.func() - const docs = await db.allDocs({ include_docs: true }) - - await migration.func() - const latestDocs = await db.allDocs({ include_docs: true }) - - expect(docs).toEqual(latestDocs) - } + expect(migrationVersion).toEqual("20231211101320_test") }) }) }) From 2a92263df5cd106ee444a3038f025e61ce962a38 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:19:07 +0100 Subject: [PATCH 15/65] Add tests --- .../appMigrations/tests/migrations.spec.ts | 45 ++++++++++++++----- .../src/tests/utilities/api/application.ts | 8 +++- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index d27a1ac37e..5eb8535695 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,17 +1,15 @@ +import { Header } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" import * as migrations from "../migrations" import { getAppMigrationVersion } from "../appMigrationMetadata" -import { AppMigration } from ".." -const mockedMigrations: AppMigration[] = [ - { - id: "20231211101320_test", - func: async () => {}, - }, -] - -jest.doMock("../migrations", () => ({ - MIGRATIONS: mockedMigrations, +jest.mock("../migrations", () => ({ + MIGRATIONS: [ + { + id: "20231211101320_test", + func: async () => {}, + }, + ], })) describe("migrations", () => { @@ -25,4 +23,31 @@ describe("migrations", () => { expect(migrationVersion).toEqual("20231211101320_test") }) }) + + it("accessing an app that has no pending migrations will not attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toBeUndefined() + }) + + it("accessing an app that has pending migrations will attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + migrations.MIGRATIONS.push({ + id: "20231211105812_new-test", + func: async () => {}, + }) + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toEqual(appId) + }) }) diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index 85bc4e4173..9c784bade1 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -1,3 +1,4 @@ +import { Response } from "supertest" import { App } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI { super(config) } - get = async (appId: string): Promise => { + getRaw = async (appId: string): Promise => { const result = await this.request .get(`/api/applications/${appId}/appPackage`) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) + return result + } + + get = async (appId: string): Promise => { + const result = await this.getRaw(appId) return result.body.application as App } } From 144fbdf94676d8db3ad52702d289886497177e4b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:30:04 +0100 Subject: [PATCH 16/65] Add tests --- packages/server/src/api/controllers/application.ts | 2 +- packages/server/src/appMigrations/index.ts | 8 +++++--- .../appMigrations/tests/migrationsProcessor.spec.ts | 12 +++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index bb4f447f79..70298c7172 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) { // Initialise the app migration version as the latest one await appMigrations.updateAppMigrationMetadata({ appId, - version: appMigrations.latestMigration, + version: appMigrations.getLatestMigrationId(), }) await cache.app.invalidateAppMetadata(appId, newApplication) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 9bc70f0abc..b382d8b533 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -12,9 +12,10 @@ export type AppMigration = { func: () => Promise } -export const latestMigration = MIGRATIONS.map(m => m.id) - .sort() - .reverse()[0] +export const getLatestMigrationId = () => + MIGRATIONS.map(m => m.id) + .sort() + .reverse()[0] const getTimestamp = (versionId: string) => versionId?.split("_")[0] @@ -24,6 +25,7 @@ export async function checkMissingMigrations( appId: string ) { const currentVersion = await getAppMigrationVersion(appId) + const latestMigration = getLatestMigrationId() if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { await queue.add( diff --git a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts index 189f6c068b..3b8e90b526 100644 --- a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts +++ b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts @@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata" import { context } from "@budibase/backend-core" import { AppMigration } from ".." +const futureTimestamp = `20500101174029` + describe("migrationsProcessor", () => { it("running migrations will update the latest applied migration", async () => { const testMigrations: AppMigration[] = [ - { id: "123", func: async () => {} }, - { id: "124", func: async () => {} }, - { id: "125", func: async () => {} }, + { id: `${futureTimestamp}_123`, func: async () => {} }, + { id: `${futureTimestamp}_124`, func: async () => {} }, + { id: `${futureTimestamp}_125`, func: async () => {} }, ] const config = setup.getConfig() @@ -23,13 +25,13 @@ describe("migrationsProcessor", () => { expect( await config.doInContext(appId, () => getAppMigrationVersion(appId)) - ).toBe("125") + ).toBe(`${futureTimestamp}_125`) }) it("no context can be initialised within a migration", async () => { const testMigrations: AppMigration[] = [ { - id: "123", + id: `${futureTimestamp}_123`, func: async () => { await context.doInAppMigrationContext("any", () => {}) }, From f3e046ba9824a1e8685e534562d54e6cda584815 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 10:21:10 +0100 Subject: [PATCH 17/65] Move header constants to shared-core --- packages/backend-core/src/constants/misc.ts | 20 +------------------ packages/shared-core/src/constants/api.ts | 19 ++++++++++++++++++ .../src/{constants.ts => constants/index.ts} | 2 ++ 3 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 packages/shared-core/src/constants/api.ts rename packages/shared-core/src/{constants.ts => constants/index.ts} (99%) diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 139bd7c0d3..aee099e10a 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -11,25 +11,7 @@ export enum Cookie { OIDC_CONFIG = "budibase:oidc:config", } -export enum Header { - API_KEY = "x-budibase-api-key", - LICENSE_KEY = "x-budibase-license-key", - API_VER = "x-budibase-api-version", - APP_ID = "x-budibase-app-id", - SESSION_ID = "x-budibase-session-id", - TYPE = "x-budibase-type", - PREVIEW_ROLE = "x-budibase-role", - TENANT_ID = "x-budibase-tenant-id", - VERIFICATION_CODE = "x-budibase-verification-code", - RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", - RESET_PASSWORD_CODE = "x-budibase-reset-password-code", - RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", - TOKEN = "x-budibase-token", - CSRF_TOKEN = "x-csrf-token", - CORRELATION_ID = "x-budibase-correlation-id", - AUTHORIZATION = "authorization", - MIGRATING_APP = "x-budibase-migrating-app", -} +export { Header } from "@budibase/shared-core" export enum GlobalRole { OWNER = "owner", diff --git a/packages/shared-core/src/constants/api.ts b/packages/shared-core/src/constants/api.ts new file mode 100644 index 0000000000..d6633649e6 --- /dev/null +++ b/packages/shared-core/src/constants/api.ts @@ -0,0 +1,19 @@ +export enum Header { + API_KEY = "x-budibase-api-key", + LICENSE_KEY = "x-budibase-license-key", + API_VER = "x-budibase-api-version", + APP_ID = "x-budibase-app-id", + SESSION_ID = "x-budibase-session-id", + TYPE = "x-budibase-type", + PREVIEW_ROLE = "x-budibase-role", + TENANT_ID = "x-budibase-tenant-id", + VERIFICATION_CODE = "x-budibase-verification-code", + RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", + RESET_PASSWORD_CODE = "x-budibase-reset-password-code", + RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", + TOKEN = "x-budibase-token", + CSRF_TOKEN = "x-csrf-token", + CORRELATION_ID = "x-budibase-correlation-id", + AUTHORIZATION = "authorization", + MIGRATING_APP = "x-budibase-migrating-app", +} diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants/index.ts similarity index 99% rename from packages/shared-core/src/constants.ts rename to packages/shared-core/src/constants/index.ts index 0787b8bed1..a23913dd11 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,3 +1,5 @@ +export * from "./api" + export const OperatorOptions = { Equals: { value: "equal", From 005dd2766366dee4d80b6b40a96a88b3ffef3797 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 12:06:55 +0100 Subject: [PATCH 18/65] Use header consts from shared-core --- packages/frontend-core/src/api/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index aefc3522a7..85296f3121 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -1,4 +1,5 @@ import { Helpers } from "@budibase/bbui" +import { Header } from "@budibase/shared-core" import { ApiVersion } from "../constants" import { buildAnalyticsEndpoints } from "./analytics" import { buildAppEndpoints } from "./app" @@ -133,9 +134,9 @@ export const createAPIClient = config => { // Build headers let headers = { Accept: "application/json" } - headers["x-budibase-session-id"] = APISessionID + headers[Header.SESSION_ID] = APISessionID if (!external) { - headers["x-budibase-api-version"] = ApiVersion + headers[Header.API_VER] = ApiVersion } if (json) { headers["Content-Type"] = "application/json" @@ -242,7 +243,7 @@ export const createAPIClient = config => { getAppID: () => { let headers = {} config?.attachHeaders(headers) - return headers?.["x-budibase-app-id"] + return headers?.[Header.APP_ID] }, } From 50da7992168bf12b4dc5645dbb792e5fed692a8e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 13:42:01 +0100 Subject: [PATCH 19/65] Add onMigrationDetected function --- packages/frontend-core/src/api/index.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 85296f3121..d4b4f3636e 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -63,6 +63,11 @@ const defaultAPIClientConfig = { * invoked before the actual JS error is thrown up the stack. */ onError: null, + + /** + * A function can be passed to be called when an API call returns info about a migration running for a specific app + */ + onMigrationDetected: null, } /** @@ -171,6 +176,7 @@ export const createAPIClient = config => { // Handle response if (response.status >= 200 && response.status < 400) { + handleMigrations(response) try { if (parseResponse) { return await parseResponse(response) @@ -187,7 +193,18 @@ export const createAPIClient = config => { } } - // Performs an API call to the server and caches the response. + const handleMigrations = response => { + if (!config.onMigrationDetected) { + return + } + const migration = response.headers.get(Header.MIGRATING_APP) + + if (migration) { + config.onMigrationDetected(migration) + } + } + + // Performs an API call to the server and caches the response. // Future invocation for this URL will return the cached result instead of // hitting the server again. const makeCachedApiCall = async params => { From 46f8f4da58d844c2022d006d9c2603c59e36ac4e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 18:37:03 +0100 Subject: [PATCH 20/65] Fix parsing appid from path with ? --- packages/backend-core/src/utils/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ee1ef6da0c..f55bd44073 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -127,7 +127,10 @@ function parseAppIdFromUrl(url?: string) { if (!url) { return } - return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX)) + return url + .split("?")[0] + .split("/") + .find(subPath => subPath.startsWith(APP_PREFIX)) } /** From 165d86c24678229607abc4e127b9b137100b1569 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 18:38:29 +0100 Subject: [PATCH 21/65] Create updating page --- .../server/src/api/controllers/migrations.ts | 26 ++++++++++++++++--- packages/server/src/api/routes/migrations.ts | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 8f1bfa22db..74e573dc6a 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,14 +1,34 @@ +import { context } from "@budibase/backend-core" import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { BBContext } from "@budibase/types" +import { Ctx } from "@budibase/types" +import { + getAppMigrationVersion, + getLatestMigrationId, +} from "../../appMigrations" -export async function migrate(ctx: BBContext) { +export async function migrate(ctx: Ctx) { const options = ctx.request.body // don't await as can take a while, just return migrationImpl(options) ctx.status = 200 } -export async function fetchDefinitions(ctx: BBContext) { +export async function fetchDefinitions(ctx: Ctx) { ctx.body = MIGRATIONS ctx.status = 200 } + +export async function migrationCompleted(ctx: Ctx) { + const appId = context.getAppId() + + if (!appId) { + ctx.throw("AppId could not be found") + } + + const latestAppliedMigration = await getAppMigrationVersion(appId) + + const migrated = latestAppliedMigration === getLatestMigrationId() + + ctx.body = { migrated } + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index f530647c78..8f5cc63f82 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -11,4 +11,6 @@ router auth.internalApi, migrationsController.fetchDefinitions ) + .get("/api/migrations/status", migrationsController.migrationCompleted) + export default router From 251663f38c62eb4de2fce99c6ed4741ec543ebab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 13 Dec 2023 09:57:12 +0100 Subject: [PATCH 22/65] Renames --- packages/server/src/api/controllers/migrations.ts | 2 +- packages/server/src/api/routes/migrations.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 74e573dc6a..c8f786578d 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -18,7 +18,7 @@ export async function fetchDefinitions(ctx: Ctx) { ctx.status = 200 } -export async function migrationCompleted(ctx: Ctx) { +export async function getMigrationStatus(ctx: Ctx) { const appId = context.getAppId() if (!appId) { diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index 8f5cc63f82..918b197de2 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -11,6 +11,6 @@ router auth.internalApi, migrationsController.fetchDefinitions ) - .get("/api/migrations/status", migrationsController.migrationCompleted) + .get("/api/migrations/status", migrationsController.getMigrationStatus) export default router From a0dd71f990d80bf9efb2b4b4114dbb2922ad9528 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:31:04 +0100 Subject: [PATCH 23/65] Renames --- packages/backend-core/src/utils/utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index f55bd44073..0554737518 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { } // look in the path - const pathId = parseAppIdFromUrl(ctx.path) + const pathId = parseAppIdFromUrlPath(ctx.path) if (!appId && pathId) { appId = confirmAppId(pathId) } @@ -116,19 +116,19 @@ export async function getAppIdFromCtx(ctx: Ctx) { // referer header is present from a builder redirect const referer = ctx.request.headers.referer if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { - const refererId = parseAppIdFromUrl(ctx.request.headers.referer) + const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer) appId = confirmAppId(refererId) } return appId } -function parseAppIdFromUrl(url?: string) { +function parseAppIdFromUrlPath(url?: string) { if (!url) { return } return url - .split("?")[0] + .split("?")[0] // Remove any possible query string .split("/") .find(subPath => subPath.startsWith(APP_PREFIX)) } From 51acf6aadc38cb3250c7dbe0f0f6a36e6b783ca7 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Dec 2023 09:47:14 +0000 Subject: [PATCH 24/65] Bump version to 2.13.40 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index ed49a4267d..5e12b57763 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.39", + "version": "2.13.40", "npmClient": "yarn", "packages": [ "packages/*", From 46abfdd1eab1277add97b71107a155bf731b293b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:48:57 +0100 Subject: [PATCH 25/65] Add label for deployments --- .github/workflows/deploy-featurebranch.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index c70f2fff20..b572908505 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -7,7 +7,10 @@ on: jobs: release: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: | + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == 'Budibase/budibase'|| + contains(github.event.issue.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 5f62004675c292805a513977859c2fcda62aeed9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:52:29 +0100 Subject: [PATCH 26/65] Fix condition --- .github/workflows/deploy-featurebranch.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index b572908505..9bbab42436 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -8,8 +8,7 @@ on: jobs: release: if: | - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == 'Budibase/budibase'|| + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') && contains(github.event.issue.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: From 934611029fd51a0f0976d33c3921530a6f62b984 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:04:28 +0100 Subject: [PATCH 27/65] Add types --- .github/workflows/deploy-featurebranch.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 9bbab42436..d177d88a2f 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,6 +2,13 @@ name: deploy-featurebranch on: pull_request: + types: [ + labeled, + # default types + opened, + synchronize, + reopened, + ] branches: - master From 651b42263be270282dee6689a548b109ab369ea4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:05:05 +0100 Subject: [PATCH 28/65] Add comment --- .github/workflows/deploy-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index d177d88a2f..b3463b4ccd 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -4,7 +4,7 @@ on: pull_request: types: [ labeled, - # default types + # default types (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) opened, synchronize, reopened, From ad7be462faae3d34308b08eaba932137ddfc773f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:09:04 +0100 Subject: [PATCH 29/65] Fix --- .github/workflows/deploy-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index b3463b4ccd..10d57bb6e3 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -16,7 +16,7 @@ jobs: release: if: | (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') && - contains(github.event.issue.labels.*.name, 'feature-branch') + contains(github.event.pull_request.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 4e934a56b7c1bd45822a6bae0fa168439876fec1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:14:05 +0100 Subject: [PATCH 30/65] Delete fb on label removed --- .github/workflows/close-featurebranch.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 46cb781730..20bf2af5d3 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -2,9 +2,7 @@ name: close-featurebranch on: pull_request: - types: [closed] - branches: - - master + types: [closed, unlabeled] workflow_dispatch: inputs: BRANCH: @@ -14,6 +12,9 @@ on: jobs: release: + if: | + (contains(github.event.pull_request.labels.*.name, 'feature-branch') || + contains(github.event.changes.labels.removed.*.name, 'feature-branch')) runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From f1190c47f8aed73fac56c805b18e64a65826d00a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:14:26 +0100 Subject: [PATCH 31/65] Do not check base on fb --- .github/workflows/deploy-featurebranch.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 10d57bb6e3..a5636fe912 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -4,13 +4,11 @@ on: pull_request: types: [ labeled, - # default types (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) + # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) opened, synchronize, reopened, ] - branches: - - master jobs: release: From c07914a0bf6cdf9ede536da37e58ad60f61a5411 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:18:35 +0100 Subject: [PATCH 32/65] Attempt close --- .github/workflows/close-featurebranch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 20bf2af5d3..e805cff0e0 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -13,8 +13,8 @@ on: jobs: release: if: | - (contains(github.event.pull_request.labels.*.name, 'feature-branch') || - contains(github.event.changes.labels.removed.*.name, 'feature-branch')) + contains(github.event.pull_request.labels.*.name, 'feature-branch') || + github.event.label.name == 'feature-branch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 8542b8d87f61fef28172f441360b67887044d5ca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:26:28 +0100 Subject: [PATCH 33/65] Do check labels --- .github/workflows/close-featurebranch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index e805cff0e0..5da3eb52cd 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -13,8 +13,8 @@ on: jobs: release: if: | - contains(github.event.pull_request.labels.*.name, 'feature-branch') || - github.event.label.name == 'feature-branch' + (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'feature-branch')) || + github.event.label.name == 'feature-branch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 7b4ab36e39ad48d93be24e7725f67e9f91dbbc7d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 10:30:17 +0000 Subject: [PATCH 34/65] Update VSCode config to adapt to new format of source.fixAll --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ece537efac..e22d5a8866 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "[json]": { From 598ab03fd070f4c9e2569abbe439d983bff79746 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 10:40:22 +0000 Subject: [PATCH 35/65] Optimise getUniqueByProp --- packages/server/src/db/linkedRows/linkUtils.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 5942e7e5a1..0d2e99fc6c 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -99,9 +99,15 @@ export async function getLinkDocuments(args: { } export function getUniqueByProp(array: any[], prop: string) { - return array.filter((obj, pos, arr) => { - return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos - }) + const seen = new Set() + const ret = [] + for (const item of array) { + if (!seen.has(item[prop])) { + seen.add(item[prop]) + ret.push(item) + } + } + return ret } export function getLinkedTableIDs(table: Table): string[] { From 0597c0efc6246a2a2388ce9235a42f939d550253 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 11:08:55 +0000 Subject: [PATCH 36/65] Respond to PR feedback. --- packages/server/src/db/linkedRows/linkUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 0d2e99fc6c..faaf9c1f21 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -100,14 +100,14 @@ export async function getLinkDocuments(args: { export function getUniqueByProp(array: any[], prop: string) { const seen = new Set() - const ret = [] + const filteredArray = [] for (const item of array) { if (!seen.has(item[prop])) { seen.add(item[prop]) - ret.push(item) + filteredArray.push(item) } } - return ret + return filteredArray } export function getLinkedTableIDs(table: Table): string[] { From 359679dfc4d1233995eb11629df48667e0c02cfe Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Dec 2023 11:34:20 +0000 Subject: [PATCH 37/65] Bump version to 2.13.41 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5e12b57763..962954db39 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.40", + "version": "2.13.41", "npmClient": "yarn", "packages": [ "packages/*", From 2cea0bb5e56ddf62c12e23e27e6223ee7324d8dd Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 14 Dec 2023 16:05:06 +0100 Subject: [PATCH 38/65] Update account portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b090705d6c..94f00563d6 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b090705d6c17029c1d0a2ff991d045de20e8e52e +Subproject commit 94f00563d68c0fa0bf534d3ca46fe728425629b3 From c5be68149203a5c016ab74ec0c31060747fbf702 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 14 Dec 2023 16:31:06 +0100 Subject: [PATCH 39/65] Update account-portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 94f00563d6..35797191e0 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 94f00563d68c0fa0bf534d3ca46fe728425629b3 +Subproject commit 35797191e000b968814fc0eaa5d02584781d7cc3 From 34545e1e86769b2c1f57d7b5999bde9f85a43d9d Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 14 Dec 2023 16:35:37 +0100 Subject: [PATCH 40/65] Update account portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 35797191e0..93c69553b1 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 35797191e000b968814fc0eaa5d02584781d7cc3 +Subproject commit 93c69553b1b948dec347f82910c6ccb97df5f79a From 995f24c64f21e551ee14e61c3028ae869e43664a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 15:35:45 +0000 Subject: [PATCH 41/65] Allow cluster internal traffic to hit the nginx status endpoint. --- hosting/proxy/nginx.prod.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 88f9645f80..65cc3ff390 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -257,6 +257,7 @@ http { access_log off; allow 127.0.0.1; + allow 10.0.0.0/8; deny all; location /nginx_status { From 7ec2c3860bfc2bb69f39d31a6b775003af39acba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 16:20:31 +0000 Subject: [PATCH 42/65] Optimise the performance of Writethrough.put. --- packages/backend-core/src/cache/writethrough.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 24e519dc7f..5cafe418d7 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -56,11 +56,8 @@ async function put( const writeDb = async (toWrite: any) => { // doc should contain the _id and _rev const response = await db.put(toWrite, { force: true }) - output = { - ...doc, - _id: response.id, - _rev: response.rev, - } + output._id = response.id + output._rev = response.rev } try { await writeDb(doc) From e7606125b61647fbac2910bff229fcb9ee61c8cf Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Dec 2023 16:33:31 +0000 Subject: [PATCH 43/65] Bump version to 2.13.42 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 962954db39..7d9ea19476 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.41", + "version": "2.13.42", "npmClient": "yarn", "packages": [ "packages/*", From 0d3ea23301e7649fa85e49728622309173a6e621 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 16:52:47 +0000 Subject: [PATCH 44/65] Instrument formula processing in DataDog. --- .../src/utilities/rowProcessor/utils.ts | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 9eb725dd7c..8663f174ff 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -11,6 +11,7 @@ import { Row, Table, } from "@budibase/types" +import tracer from "dd-trace" interface FormulaOpts { dynamic?: boolean @@ -50,33 +51,40 @@ export function processFormulas( inputRows: T, { dynamic, contextRows }: FormulaOpts = { dynamic: true } ): T { - const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - if (rows) - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + return tracer.trace("processFormulas", {}, span => { + span?.addTags({ tableId: table._id }) + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) { + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rows.length; i++) { - let row = rows[i] - let context = contextRows ? contextRows[i] : row - rows[i] = { - ...row, - [column]: processStringSync(schema.formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + let formula = schema.formula + rows[i] = { + ...row, + [column]: tracer.trace("processStringSync", {}, () => + processStringSync(formula, context) + ), + } } } } - return Array.isArray(inputRows) ? rows : rows[0] + return Array.isArray(inputRows) ? rows : rows[0] + }) } /** From 899b6707e75628ffc22fac04dbaff35ad7a3f04b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 17:10:09 +0000 Subject: [PATCH 45/65] add the column into the processStringSync span --- packages/server/src/utilities/rowProcessor/utils.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 8663f174ff..ddffb0eca9 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -76,9 +76,10 @@ export function processFormulas( let formula = schema.formula rows[i] = { ...row, - [column]: tracer.trace("processStringSync", {}, () => - processStringSync(formula, context) - ), + [column]: tracer.trace("processStringSync", {}, span => { + span?.addTags({ column }) + return processStringSync(formula, context) + }), } } } From 5b880f1d9bf531cca0bcf0ea7d2fe204152b30bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 17:26:48 +0000 Subject: [PATCH 46/65] Also instrument appId --- packages/server/src/middleware/currentapp.ts | 175 ++++++++++--------- 1 file changed, 92 insertions(+), 83 deletions(-) diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index 984dd8e5e9..debacabf66 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -12,103 +12,112 @@ import { getCachedSelf } from "../utilities/global" import env from "../environment" import { isWebhookEndpoint } from "./utils" import { UserCtx, ContextUser } from "@budibase/types" +import tracer from "dd-trace" export default async (ctx: UserCtx, next: any) => { - // try to get the appID from the request - let requestAppId = await utils.getAppIdFromCtx(ctx) - if (!requestAppId) { - return next() - } - - // deny access to application preview - if (!env.isTest()) { - if ( - isDevAppID(requestAppId) && - !isWebhookEndpoint(ctx) && - !users.isBuilder(ctx.user, requestAppId) - ) { - return ctx.redirect("/") + return tracer.trace("currentapp middleware", {}, async span => { + // try to get the appID from the request + let requestAppId = await utils.getAppIdFromCtx(ctx) + if (!requestAppId) { + return next() } - } - let appId: string | undefined, - roleId = roles.BUILTIN_ROLE_IDS.PUBLIC - if (!ctx.user?._id) { - // not logged in, try to set a cookie for public apps - appId = requestAppId - } else if (requestAppId != null) { - // Different App ID means cookie needs reset, or if the same public user has logged in - const globalUser = await getCachedSelf(ctx, requestAppId) - appId = requestAppId - // retrieving global user gets the right role - roleId = globalUser.roleId || roleId + if (requestAppId) { + span?.addTags({ appId: requestAppId }) + } - // Allow builders to specify their role via a header - const isBuilder = users.isBuilder(globalUser, appId) - const isDevApp = appId && isDevAppID(appId) - const roleHeader = - ctx.request && - (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string) - if (isBuilder && isDevApp && roleHeader) { - // Ensure the role is valid by ensuring a definition exists - try { - if (roleHeader) { - await roles.getRole(roleHeader) - roleId = roleHeader + // deny access to application preview + if (!env.isTest()) { + if ( + isDevAppID(requestAppId) && + !isWebhookEndpoint(ctx) && + !users.isBuilder(ctx.user, requestAppId) + ) { + return ctx.redirect("/") + } + } - // Delete admin and builder flags so that the specified role is honoured - ctx.user = users.removePortalUserPermissions(ctx.user) as ContextUser + let appId: string | undefined, + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + if (!ctx.user?._id) { + // not logged in, try to set a cookie for public apps + appId = requestAppId + } else if (requestAppId != null) { + // Different App ID means cookie needs reset, or if the same public user has logged in + const globalUser = await getCachedSelf(ctx, requestAppId) + appId = requestAppId + // retrieving global user gets the right role + roleId = globalUser.roleId || roleId + + // Allow builders to specify their role via a header + const isBuilder = users.isBuilder(globalUser, appId) + const isDevApp = appId && isDevAppID(appId) + const roleHeader = + ctx.request && + (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string) + if (isBuilder && isDevApp && roleHeader) { + // Ensure the role is valid by ensuring a definition exists + try { + if (roleHeader) { + await roles.getRole(roleHeader) + roleId = roleHeader + + // Delete admin and builder flags so that the specified role is honoured + ctx.user = users.removePortalUserPermissions( + ctx.user + ) as ContextUser + } + } catch (error) { + // Swallow error and do nothing } - } catch (error) { - // Swallow error and do nothing } } - } - // nothing more to do - if (!appId) { - return next() - } + // nothing more to do + if (!appId) { + return next() + } - const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined + const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined - // if the user is not in the right tenant then make sure to wipe their cookie - // also cleanse any information about them that has been allocated - // this avoids apps making calls to say the worker which are cross tenant, - // we simply remove the authentication - if ( - env.MULTI_TENANCY && - userId && - requestAppId && - !tenancy.isUserInAppTenant(requestAppId, ctx.user) - ) { - // clear out the user - ctx.user = users.cleanseUserObject(ctx.user) as ContextUser - ctx.isAuthenticated = false - roleId = roles.BUILTIN_ROLE_IDS.PUBLIC - // remove the cookie, so future calls are public - await auth.platformLogout({ - ctx, - userId, - }) - } - - return context.doInAppContext(appId, async () => { - ctx.appId = appId - if (roleId) { - ctx.roleId = roleId - const globalId = ctx.user ? ctx.user._id : undefined - ctx.user = { - ...ctx.user!, - // override userID with metadata one - _id: userId, + // if the user is not in the right tenant then make sure to wipe their cookie + // also cleanse any information about them that has been allocated + // this avoids apps making calls to say the worker which are cross tenant, + // we simply remove the authentication + if ( + env.MULTI_TENANCY && + userId && + requestAppId && + !tenancy.isUserInAppTenant(requestAppId, ctx.user) + ) { + // clear out the user + ctx.user = users.cleanseUserObject(ctx.user) as ContextUser + ctx.isAuthenticated = false + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + // remove the cookie, so future calls are public + await auth.platformLogout({ + ctx, userId, - globalId, - roleId, - role: await roles.getRole(roleId, { defaultPublic: true }), - } + }) } - return next() + return context.doInAppContext(appId, async () => { + ctx.appId = appId + if (roleId) { + ctx.roleId = roleId + const globalId = ctx.user ? ctx.user._id : undefined + ctx.user = { + ...ctx.user!, + // override userID with metadata one + _id: userId, + userId, + globalId, + roleId, + role: await roles.getRole(roleId, { defaultPublic: true }), + } + } + + return next() + }) }) } From a6f03b02f24d1ef99c61df74601fb0102d1dec43 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 17:27:22 +0000 Subject: [PATCH 47/65] Use the right casing for DataDog --- packages/server/src/middleware/currentapp.ts | 2 +- packages/server/src/utilities/rowProcessor/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index debacabf66..e13982a624 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -23,7 +23,7 @@ export default async (ctx: UserCtx, next: any) => { } if (requestAppId) { - span?.addTags({ appId: requestAppId }) + span?.addTags({ app_id: requestAppId }) } // deny access to application preview diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index ddffb0eca9..d0072053b8 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -52,7 +52,7 @@ export function processFormulas( { dynamic, contextRows }: FormulaOpts = { dynamic: true } ): T { return tracer.trace("processFormulas", {}, span => { - span?.addTags({ tableId: table._id }) + span?.addTags({ table_id: table._id }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] if (rows) { for (let [column, schema] of Object.entries(table.schema)) { From aa5ea53af2dd2f56dd82aac73dd3a869c850ee5e Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Fri, 15 Dec 2023 08:36:30 +0100 Subject: [PATCH 48/65] Update account portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 93c69553b1..b47ad3f331 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 93c69553b1b948dec347f82910c6ccb97df5f79a +Subproject commit b47ad3f33177345b9a1685f5dbc10953c8c1c7cc From d0bf82ea2dfebed988822013767c1efeda6d99b2 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 15 Dec 2023 08:14:36 +0000 Subject: [PATCH 49/65] Bump version to 2.13.43 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7d9ea19476..0ed2de613f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.42", + "version": "2.13.43", "npmClient": "yarn", "packages": [ "packages/*", From 56a240bc187428b42804dd0062c86ebda14c61d2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 09:53:48 +0000 Subject: [PATCH 50/65] Set up DD version and code tracking. --- packages/server/Dockerfile | 4 ++++ packages/worker/Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index f737570fcd..2af37ae288 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -68,9 +68,13 @@ COPY packages/server/builder/ builder/ COPY packages/server/client/ client/ ARG BUDIBASE_VERSION +ARG GIT_COMMIT_SHA # Ensuring the version argument is sent RUN test -n "$BUDIBASE_VERSION" ENV BUDIBASE_VERSION=$BUDIBASE_VERSION +ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase +ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA +ENV DD_VERSION=$BUDIBASE_VERSION EXPOSE 4001 diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 4706ca155a..b03a817c8b 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -51,8 +51,12 @@ ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR ENV ACCOUNT_PORTAL_URL=https://account.budibase.app ARG BUDIBASE_VERSION +ARG GIT_COMMIT_SHA # Ensuring the version argument is sent RUN test -n "$BUDIBASE_VERSION" ENV BUDIBASE_VERSION=$BUDIBASE_VERSION +ENV DD_GIT_REPOSITORY_URL=https://github.com/budibase/budibase +ENV DD_GIT_COMMIT_SHA=$GIT_COMMIT_SHA +ENV DD_VERSION=$BUDIBASE_VERSION CMD ["./docker_run.sh"] From 7a11c5be92dfef4484886d50f45d6800c29309f5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 11:10:23 +0000 Subject: [PATCH 51/65] Tweak newly added spans to track formula processing. --- packages/server/src/middleware/currentapp.ts | 191 +++++++++--------- .../src/utilities/rowProcessor/utils.ts | 5 +- 2 files changed, 97 insertions(+), 99 deletions(-) diff --git a/packages/server/src/middleware/currentapp.ts b/packages/server/src/middleware/currentapp.ts index e13982a624..4b11b933af 100644 --- a/packages/server/src/middleware/currentapp.ts +++ b/packages/server/src/middleware/currentapp.ts @@ -15,109 +15,106 @@ import { UserCtx, ContextUser } from "@budibase/types" import tracer from "dd-trace" export default async (ctx: UserCtx, next: any) => { - return tracer.trace("currentapp middleware", {}, async span => { - // try to get the appID from the request - let requestAppId = await utils.getAppIdFromCtx(ctx) - if (!requestAppId) { - return next() - } + // try to get the appID from the request + let requestAppId = await utils.getAppIdFromCtx(ctx) + if (!requestAppId) { + return next() + } - if (requestAppId) { - span?.addTags({ app_id: requestAppId }) - } + if (requestAppId) { + const span = tracer.scope().active() + span?.addTags({ app_id: requestAppId }) + } - // deny access to application preview - if (!env.isTest()) { - if ( - isDevAppID(requestAppId) && - !isWebhookEndpoint(ctx) && - !users.isBuilder(ctx.user, requestAppId) - ) { - return ctx.redirect("/") - } - } - - let appId: string | undefined, - roleId = roles.BUILTIN_ROLE_IDS.PUBLIC - if (!ctx.user?._id) { - // not logged in, try to set a cookie for public apps - appId = requestAppId - } else if (requestAppId != null) { - // Different App ID means cookie needs reset, or if the same public user has logged in - const globalUser = await getCachedSelf(ctx, requestAppId) - appId = requestAppId - // retrieving global user gets the right role - roleId = globalUser.roleId || roleId - - // Allow builders to specify their role via a header - const isBuilder = users.isBuilder(globalUser, appId) - const isDevApp = appId && isDevAppID(appId) - const roleHeader = - ctx.request && - (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string) - if (isBuilder && isDevApp && roleHeader) { - // Ensure the role is valid by ensuring a definition exists - try { - if (roleHeader) { - await roles.getRole(roleHeader) - roleId = roleHeader - - // Delete admin and builder flags so that the specified role is honoured - ctx.user = users.removePortalUserPermissions( - ctx.user - ) as ContextUser - } - } catch (error) { - // Swallow error and do nothing - } - } - } - - // nothing more to do - if (!appId) { - return next() - } - - const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined - - // if the user is not in the right tenant then make sure to wipe their cookie - // also cleanse any information about them that has been allocated - // this avoids apps making calls to say the worker which are cross tenant, - // we simply remove the authentication + // deny access to application preview + if (!env.isTest()) { if ( - env.MULTI_TENANCY && - userId && - requestAppId && - !tenancy.isUserInAppTenant(requestAppId, ctx.user) + isDevAppID(requestAppId) && + !isWebhookEndpoint(ctx) && + !users.isBuilder(ctx.user, requestAppId) ) { - // clear out the user - ctx.user = users.cleanseUserObject(ctx.user) as ContextUser - ctx.isAuthenticated = false - roleId = roles.BUILTIN_ROLE_IDS.PUBLIC - // remove the cookie, so future calls are public - await auth.platformLogout({ - ctx, + return ctx.redirect("/") + } + } + + let appId: string | undefined, + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + if (!ctx.user?._id) { + // not logged in, try to set a cookie for public apps + appId = requestAppId + } else if (requestAppId != null) { + // Different App ID means cookie needs reset, or if the same public user has logged in + const globalUser = await getCachedSelf(ctx, requestAppId) + appId = requestAppId + // retrieving global user gets the right role + roleId = globalUser.roleId || roleId + + // Allow builders to specify their role via a header + const isBuilder = users.isBuilder(globalUser, appId) + const isDevApp = appId && isDevAppID(appId) + const roleHeader = + ctx.request && + (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string) + if (isBuilder && isDevApp && roleHeader) { + // Ensure the role is valid by ensuring a definition exists + try { + if (roleHeader) { + await roles.getRole(roleHeader) + roleId = roleHeader + + // Delete admin and builder flags so that the specified role is honoured + ctx.user = users.removePortalUserPermissions(ctx.user) as ContextUser + } + } catch (error) { + // Swallow error and do nothing + } + } + } + + // nothing more to do + if (!appId) { + return next() + } + + const userId = ctx.user ? generateUserMetadataID(ctx.user._id!) : undefined + + // if the user is not in the right tenant then make sure to wipe their cookie + // also cleanse any information about them that has been allocated + // this avoids apps making calls to say the worker which are cross tenant, + // we simply remove the authentication + if ( + env.MULTI_TENANCY && + userId && + requestAppId && + !tenancy.isUserInAppTenant(requestAppId, ctx.user) + ) { + // clear out the user + ctx.user = users.cleanseUserObject(ctx.user) as ContextUser + ctx.isAuthenticated = false + roleId = roles.BUILTIN_ROLE_IDS.PUBLIC + // remove the cookie, so future calls are public + await auth.platformLogout({ + ctx, + userId, + }) + } + + return context.doInAppContext(appId, async () => { + ctx.appId = appId + if (roleId) { + ctx.roleId = roleId + const globalId = ctx.user ? ctx.user._id : undefined + ctx.user = { + ...ctx.user!, + // override userID with metadata one + _id: userId, userId, - }) + globalId, + roleId, + role: await roles.getRole(roleId, { defaultPublic: true }), + } } - return context.doInAppContext(appId, async () => { - ctx.appId = appId - if (roleId) { - ctx.roleId = roleId - const globalId = ctx.user ? ctx.user._id : undefined - ctx.user = { - ...ctx.user!, - // override userID with metadata one - _id: userId, - userId, - globalId, - roleId, - role: await roles.getRole(roleId, { defaultPublic: true }), - } - } - - return next() - }) + return next() }) } diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index d0072053b8..431c83cb36 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -52,7 +52,8 @@ export function processFormulas( { dynamic, contextRows }: FormulaOpts = { dynamic: true } ): T { return tracer.trace("processFormulas", {}, span => { - span?.addTags({ table_id: table._id }) + const numRows = Array.isArray(inputRows) ? inputRows.length : 1 + span?.addTags({ table_id: table._id, dynamic, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] if (rows) { for (let [column, schema] of Object.entries(table.schema)) { @@ -77,7 +78,7 @@ export function processFormulas( rows[i] = { ...row, [column]: tracer.trace("processStringSync", {}, span => { - span?.addTags({ column }) + span?.addTags({ table_id: table._id, column, statis: isStatic }) return processStringSync(formula, context) }), } From daf669b0e32b3c79bf87957722feda542d7fb16b Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Fri, 15 Dec 2023 12:44:42 +0100 Subject: [PATCH 52/65] Upgrade account portal submodule -> plan list fix --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b47ad3f331..56e4283999 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b47ad3f33177345b9a1685f5dbc10953c8c1c7cc +Subproject commit 56e4283999dfe6356682dc280b2010d7a6a5e594 From fb9a7773189f80a821d6f7d841ba0135c367d0d0 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Fri, 15 Dec 2023 12:53:27 +0100 Subject: [PATCH 53/65] Update account-portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 56e4283999..ba0a2eb051 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 56e4283999dfe6356682dc280b2010d7a6a5e594 +Subproject commit ba0a2eb0515450f4c41ba41dc9ec72995944e5f4 From c4f124c78a03009f49ddd46176bb2d9fe0dfa2a8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 12:22:37 +0000 Subject: [PATCH 54/65] Update utils.ts Co-authored-by: Adria Navarro --- packages/server/src/utilities/rowProcessor/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 431c83cb36..cafd366cae 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -78,7 +78,7 @@ export function processFormulas( rows[i] = { ...row, [column]: tracer.trace("processStringSync", {}, span => { - span?.addTags({ table_id: table._id, column, statis: isStatic }) + span?.addTags({ table_id: table._id, column, static: isStatic }) return processStringSync(formula, context) }), } From 92f59115a09cf0a1404c82979136f6fb31d0df11 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Fri, 15 Dec 2023 15:34:21 +0100 Subject: [PATCH 55/65] Upgrade submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index ba0a2eb051..abb2c327e4 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ba0a2eb0515450f4c41ba41dc9ec72995944e5f4 +Subproject commit abb2c327e487012ba598b69836bbd25f4c45f8c8 From 61d360b8c9b9cd3429d4a89fc26901935b916901 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 15 Dec 2023 14:41:06 +0000 Subject: [PATCH 56/65] Bump version to 2.13.44 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 0ed2de613f..7462903994 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.43", + "version": "2.13.44", "npmClient": "yarn", "packages": [ "packages/*", From f27e5e96f26968343fe2798da3e82a98951bcc39 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Fri, 15 Dec 2023 16:23:30 +0100 Subject: [PATCH 57/65] Upgrade submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index abb2c327e4..0f59871826 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit abb2c327e487012ba598b69836bbd25f4c45f8c8 +Subproject commit 0f5987182652652e274c9b134abe0c87e6edf7ef From 92a230667fcd3ae71759c570bcc3463dae5a976d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 16:27:20 +0000 Subject: [PATCH 58/65] Tell a user if their JS failed due to a timeout. --- packages/string-templates/src/helpers/javascript.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index c5996c25f0..d757408780 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -56,6 +56,9 @@ module.exports.processJS = (handlebars, context) => { const res = { data: runJS(js, sandboxContext) } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error) { + if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") { + return "Timed out while executing JS" + } return "Error while executing JS" } } From b963e7081e55e5dab19433ad5a9204d96f7fb5b8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 16:50:36 +0000 Subject: [PATCH 59/65] Fix test. --- packages/string-templates/src/helpers/javascript.js | 1 + packages/string-templates/test/javascript.spec.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.js index d757408780..53baec8613 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.js @@ -56,6 +56,7 @@ module.exports.processJS = (handlebars, context) => { const res = { data: runJS(js, sandboxContext) } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` } catch (error) { + console.log(`JS error: ${typeof error} ${JSON.stringify(error)}`) if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") { return "Timed out while executing JS" } diff --git a/packages/string-templates/test/javascript.spec.js b/packages/string-templates/test/javascript.spec.js index bbc1cce246..652f7ac1cd 100644 --- a/packages/string-templates/test/javascript.spec.js +++ b/packages/string-templates/test/javascript.spec.js @@ -114,7 +114,7 @@ describe("Test the JavaScript helper", () => { it("should timeout after one second", () => { const output = processJS(`while (true) {}`) - expect(output).toBe("Error while executing JS") + expect(output).toBe("Timed out while executing JS") }) it("should prevent access to the process global", () => { From e46140cb3afa7c619c054bc2429bfddafcb8427a Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 18 Dec 2023 09:36:11 +0100 Subject: [PATCH 60/65] - Avoid BUSINESS plan usage in string literals. - Change PREMIUM_PLUS by PREMIUM - Change ENTERPRISE_BASIC by ENTERPRISE --- .../builder/portal/account/auditLogs/index.svelte | 2 +- .../src/pages/builder/portal/account/usage.svelte | 10 +++++++++- .../pages/builder/portal/settings/auth/index.svelte | 2 +- .../builder/portal/settings/environment/index.svelte | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte index a5b0b19c9b..4b68c285f8 100644 --- a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte @@ -257,7 +257,7 @@ { diff --git a/packages/builder/src/pages/builder/portal/account/usage.svelte b/packages/builder/src/pages/builder/portal/account/usage.svelte index cbe84973ef..9e7c402e91 100644 --- a/packages/builder/src/pages/builder/portal/account/usage.svelte +++ b/packages/builder/src/pages/builder/portal/account/usage.svelte @@ -15,6 +15,7 @@ import { DashCard, Usage } from "components/usage" import { PlanModel } from "constants" import { sdk } from "@budibase/shared-core" + import { PlanType } from "@budibase/types" let staticUsage = [] let monthlyUsage = [] @@ -106,7 +107,14 @@ } const planTitle = () => { - return `${capitalise(license?.plan.type)} Plan` + const planType = license?.plan.type + let planName = license?.plan.type + if (planType === PlanType.PREMIUM_PLUS) { + planName = "Premium" + } else if (planType === PlanType.ENTERPRISE_BASIC) { + planName = "Enterprise" + } + return `${capitalise(planName)} Plan` } const getDaysRemaining = timestamp => { diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 36cf5c13a8..8c7cad7eeb 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -283,7 +283,7 @@ {#if !$licensing.enforceableSSO} - Enterprise + Enterprise plan {/if} diff --git a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte index cff578febd..0d4c7e1479 100644 --- a/packages/builder/src/pages/builder/portal/settings/environment/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/environment/index.svelte @@ -59,7 +59,7 @@ { From 0d56cb8cff7173a66aa9172b3080992742e5f523 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 18 Dec 2023 09:37:18 +0100 Subject: [PATCH 61/65] Upgrade account portal submodule --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b47ad3f331..a1832145f0 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b47ad3f33177345b9a1685f5dbc10953c8c1c7cc +Subproject commit a1832145f08fca9b923cc145930d5d3a4c2f1d1f From 70636224779a7f5c95d50dcd9d0257b24f18ea94 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Dec 2023 09:43:41 +0100 Subject: [PATCH 62/65] Update account-portal ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 0f59871826..a1832145f0 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 0f5987182652652e274c9b134abe0c87e6edf7ef +Subproject commit a1832145f08fca9b923cc145930d5d3a4c2f1d1f From a855b289a25339add6b87d056f1fb398b7abf6f9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Dec 2023 09:47:26 +0100 Subject: [PATCH 63/65] Don't check submodule refs for oss references --- .github/workflows/budibase_ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4b241c5165..14cc110214 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -204,7 +204,7 @@ jobs: check-pro-submodule: runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -254,7 +254,7 @@ jobs: check-accountportal-submodule: runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules uses: actions/checkout@v3 From c5fa0806c82f1e6b81a5dfd91e7c0dd3f7c435ab Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 18 Dec 2023 09:20:44 +0000 Subject: [PATCH 64/65] Optimise getLinkDocuments --- packages/server/src/db/linkedRows/linkUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index faaf9c1f21..20f752407f 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -56,7 +56,7 @@ export async function getLinkDocuments(args: { try { let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows // filter to get unique entries - const foundIds: string[] = [] + const foundIds = new Set() linkRows = linkRows.filter(link => { // make sure anything unique is the correct key if ( @@ -65,9 +65,9 @@ export async function getLinkDocuments(args: { ) { return false } - const unique = foundIds.indexOf(link.id) === -1 + const unique = !foundIds.has(link.id) if (unique) { - foundIds.push(link.id) + foundIds.add(link.id) } return unique }) From 31181d8186914c953c596a0aa7ce3e9787e11c05 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 18 Dec 2023 09:51:14 +0000 Subject: [PATCH 65/65] Bump version to 2.13.45 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7462903994..6a7fb47923 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.44", + "version": "2.13.45", "npmClient": "yarn", "packages": [ "packages/*",