diff --git a/.all-contributorsrc b/.all-contributorsrc
index 53705907c2..3a416f917e 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -162,6 +162,7 @@
"translation"
]
},
+ {
"login": "mslourens",
"name": "Maurits Lourens",
"avatar_url": "https://avatars.githubusercontent.com/u/1907152?v=4",
diff --git a/.eslintignore b/.eslintignore
index 54824be5c7..1dac74b117 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -3,8 +3,11 @@ public
dist
packages/server/builder
packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
packages/server/client
packages/builder/.routify
packages/builder/cypress/support/queryLevelTransformerFunction.js
packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
-packages/builder/cypress/reports
\ No newline at end of file
+packages/builder/cypress/reports
+packages/sdk/sdk
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/epic.md b/.github/ISSUE_TEMPLATE/epic.md
new file mode 100644
index 0000000000..b8cf652125
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/epic.md
@@ -0,0 +1,24 @@
+---
+name: Epic
+about: Plan a new project
+title: ''
+labels: epic
+assignees: ''
+
+---
+
+## Description
+Brief summary of what this Epic is, whether it's a larger project, goal, or user story. Describe the job to be done, which persona this Epic is mainly for, or if more multiple, break it down by user and job story.
+
+## Spec
+Link to confluence spec
+
+## Teams and Stakeholders
+Describe who needs to be kept up-to-date about this Epic, included in discussions, or updated along the way. Stakeholders can be both in Product/Engineering, as well as other teams like Customer Success who might want to keep customers updated on the Epic project.
+
+
+## Workflow
+- [ ] Spec Created and pasted above
+- [ ] Product Review
+- [ ] Designs created
+- [ ] Individual Tasks created and assigned to Epic
diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index e940e6fa10..c07f9b2c28 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -23,6 +23,15 @@ jobs:
build:
runs-on: ubuntu-latest
+ services:
+ couchdb:
+ image: ibmcom/couchdb3
+ env:
+ COUCHDB_PASSWORD: budibase
+ COUCHDB_USER: budibase
+ ports:
+ - 4567:5984
+
strategy:
matrix:
node-version: [14.x]
@@ -49,13 +58,12 @@ jobs:
- uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
- files: ./packages/server/coverage/clover.xml
+ files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
name: codecov-umbrella
verbose: true
- # TODO: parallelise this
- - name: Cypress run
- uses: cypress-io/github-action@v2
- with:
- install: false
- command: yarn test:e2e:ci
+ - name: QA Core Integration Tests
+ run: |
+ cd qa-core
+ yarn
+ yarn api:test:ci
\ No newline at end of file
diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml
index 0fb8a5fea0..b37ff9cee8 100644
--- a/.github/workflows/deploy-release.yml
+++ b/.github/workflows/deploy-release.yml
@@ -69,6 +69,28 @@ jobs:
env:
KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
+ - name: Re roll app-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment app-service -n budibase
+
+ - name: Re roll proxy-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment proxy-service -n budibase
+
+ - name: Re roll worker-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment worker-service -n budibase
+
+
- name: Discord Webhook Action
uses: tsickert/discord-webhook@v4.0.0
with:
diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml
index 8bf8f232c5..cd16574eea 100644
--- a/.github/workflows/deploy-single-image.yml
+++ b/.github/workflows/deploy-single-image.yml
@@ -4,8 +4,6 @@ on:
workflow_dispatch:
env:
- BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
- BRANCH: ${{ github.event.pull_request.head.ref }}
CI: true
PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
REGISTRY_URL: registry.hub.docker.com
@@ -17,6 +15,11 @@ jobs:
matrix:
node-version: [14.x]
steps:
+ - name: Fail if branch is not master
+ if: github.ref != 'refs/heads/master'
+ run: |
+ echo "Ref is not master, you must run this job from master."
+ exit 1
- name: "Checkout"
uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-version }}
@@ -28,8 +31,6 @@ jobs:
- name: Setup Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
- - name: Install Pro
- run: yarn install:pro $BRANCH $BASE_BRANCH
- name: Run Yarn
run: yarn
- name: Run Yarn Bootstrap
diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml
index 631308d945..21c74851e1 100644
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@@ -18,8 +18,9 @@ on:
workflow_dispatch:
env:
- # Posthog token used by ui at build time
- POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
+ # Posthog token used by ui at build time
+ # disable unless needed for testing
+ # POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
FEATURE_PREVIEW_URL: https://budirelease.live
@@ -45,7 +46,8 @@ jobs:
- run: yarn
- run: yarn bootstrap
- run: yarn lint
- - run: yarn build
+ - run: yarn build
+ - run: yarn build:sdk
- run: yarn test
- name: Configure AWS Credentials
@@ -119,6 +121,27 @@ jobs:
]
env:
KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
+
+ - name: Re roll app-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment app-service -n budibase
+
+ - name: Re roll proxy-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment proxy-service -n budibase
+
+ - name: Re roll worker-service
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
+ with:
+ args: rollout restart deployment worker-service -n budibase
- name: Discord Webhook Action
uses: tsickert/discord-webhook@v4.0.0
diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml
index fc2b7b0cca..d78180fdc7 100644
--- a/.github/workflows/release-selfhost.yml
+++ b/.github/workflows/release-selfhost.yml
@@ -8,19 +8,28 @@ jobs:
runs-on: ubuntu-latest
steps:
+ - name: Fail if branch is not master
+ if: github.ref != 'refs/heads/master'
+ run: |
+ echo "Ref is not master, you must run this job from master."
+ exit 1
+
- uses: actions/checkout@v2
with:
node-version: 14.x
fetch_depth: 0
+ - name: Get the latest budibase release version
+ id: version
+ run: |
+ release_version=$(cat lerna.json | jq -r '.version')
+ echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
- name: Tag and release Docker images (Self Host)
run: |
docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
- # Get latest release version
- release_version=$(cat lerna.json | jq -r '.version')
- echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
- release_tag=v$release_version
+ release_tag=v${{ env.RELEASE_VERSION }}
# Pull apps and worker images
docker pull budibase/apps:$release_tag
@@ -40,13 +49,12 @@ jobs:
DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
SELFHOST_TAG: latest
-
- - name: Build CLI executables
+
+ - name: Bootstrap and build (CLI)
run: |
- pushd packages/cli
yarn
+ yarn bootstrap
yarn build
- popd
- name: Build OpenAPI spec
run: |
@@ -93,4 +101,4 @@ jobs:
with:
webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host."
- embed-title: ${{ env.RELEASE_VERSION }}
\ No newline at end of file
+ embed-title: ${{ env.RELEASE_VERSION }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 348b600f90..de288dd7db 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -29,7 +29,7 @@ on:
env:
# Posthog token used by ui at build time
- POSTHOG_TOKEN: phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
+ POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
@@ -56,6 +56,7 @@ jobs:
- run: yarn bootstrap
- run: yarn lint
- run: yarn build
+ - run: yarn build:sdk
- run: yarn test
- name: Configure AWS Credentials
diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml
index 7002c8335b..cffb914aaf 100644
--- a/.github/workflows/smoke_test.yaml
+++ b/.github/workflows/smoke_test.yaml
@@ -1,4 +1,4 @@
-name: Budibase Smoke Test
+name: Budibase Nightly Tests
on:
workflow_dispatch:
@@ -6,7 +6,7 @@ on:
- cron: "0 5 * * *" # every day at 5AM
jobs:
- release:
+ nightly:
runs-on: ubuntu-latest
steps:
@@ -43,6 +43,18 @@ jobs:
name: Test Reports
path: packages/builder/cypress/reports/testReport.html
+ # TODO: enable once running in QA test env
+ # - name: Configure AWS Credentials
+ # uses: aws-actions/configure-aws-credentials@v1
+ # with:
+ # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ # aws-region: eu-west-1
+
+ # - name: Upload test results HTML
+ # uses: aws-actions/configure-aws-credentials@v1
+ # run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html
+
- name: Cypress Discord Notify
run: yarn test:e2e:ci:notify
env:
diff --git a/.gitignore b/.gitignore
index f063e2224f..e1d3e6db0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,6 +63,7 @@ typings/
# dotenv environment variables file
.env
+!qa-core/.env
!hosting/.env
hosting/.generated-nginx.dev.conf
hosting/proxy/.generated-nginx.prod.conf
@@ -102,4 +103,6 @@ packages/builder/cypress/reports
stats.html
# TypeScript cache
-*.tsbuildinfo
\ No newline at end of file
+*.tsbuildinfo
+budibase-component
+budibase-datasource
diff --git a/.prettierignore b/.prettierignore
index bbeff65da7..7eb567d517 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -4,8 +4,11 @@ dist
packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
packages/server/builder
packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
packages/server/client
packages/server/src/definitions/openapi.ts
packages/builder/.routify
packages/builder/cypress/support/queryLevelTransformerFunction.js
packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
+packages/sdk/sdk
\ No newline at end of file
diff --git a/.prettierrc.json b/.prettierrc.json
index 39654fd9f9..dae5906124 100644
--- a/.prettierrc.json
+++ b/.prettierrc.json
@@ -4,7 +4,7 @@
"singleQuote": false,
"trailingComma": "es5",
"arrowParens": "avoid",
- "jsxBracketSameLine": false,
+ "bracketSameLine": false,
"plugins": ["prettier-plugin-svelte"],
"svelteSortOrder": "options-scripts-markup-styles"
}
diff --git a/README.md b/README.md
index 1dec1737da..bd38610566 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ Budibase is open-source - licensed as GPL v3. This should fill you with confiden
### Load data or start from scratch
-Budibase pulls in data from multiple sources, including MongoDB, CouchDB, PostgreSQL, MySQL, Airtable, S3, DynamoDB, or a REST API. And unlike other platforms, with Budibase you can start from scratch and create business apps with no data sources. [Request new data sources](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
+Budibase pulls in data from multiple sources, including MongoDB, CouchDB, PostgreSQL, MySQL, Airtable, S3, DynamoDB, or a REST API. And unlike other platforms, with Budibase you can start from scratch and create business apps with no datasources. [Request new datasources](https://github.com/Budibase/budibase/discussions?discussions_q=category%3AIdeas).
diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index fd46e77647..72d9fc93a9 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -60,8 +60,6 @@ spec:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: jwtSecret
- - name: LOG_LEVEL
- value: {{ .Values.services.apps.logLevel | default "info" | quote }}
{{ if .Values.services.objectStore.region }}
- name: AWS_REGION
value: {{ .Values.services.objectStore.region }}
@@ -78,6 +76,14 @@ spec:
key: objectStoreSecret
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
+ - name: PLUGIN_BUCKET_NAME
+ value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+ - name: APPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.appsBucketName | quote }}
+ - name: GLOBAL_CLOUD_BUCKET_NAME
+ value: {{ .Values.services.objectStore.globalBucketName | quote }}
+ - name: BACKUPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.apps.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}
@@ -124,12 +130,34 @@ spec:
value: {{ .Values.globals.tenantFeatureFlags | quote }}
{{ if .Values.globals.bbAdminUserEmail }}
- name: BB_ADMIN_USER_EMAIL
- value: { { .Values.globals.bbAdminUserEmail | quote } }
+ value: {{ .Values.globals.bbAdminUserEmail | quote }}
{{ end }}
{{ if .Values.globals.bbAdminUserPassword }}
- name: BB_ADMIN_USER_PASSWORD
- value: { { .Values.globals.bbAdminUserPassword | quote } }
+ value: {{ .Values.globals.bbAdminUserPassword | quote }}
{{ end }}
+ {{ if .Values.globals.pluginsDir }}
+ - name: PLUGINS_DIR
+ value: {{ .Values.globals.pluginsDir | quote }}
+ {{ end }}
+ {{ if .Values.services.apps.nodeDebug }}
+ - name: NODE_DEBUG
+ value: {{ .Values.services.apps.nodeDebug | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmEnabled }}
+ - name: ELASTIC_APM_ENABLED
+ value: {{ .Values.globals.elasticApmEnabled | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmSecretToken }}
+ - name: ELASTIC_APM_SECRET_TOKEN
+ value: {{ .Values.globals.elasticApmSecretToken | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmServerUrl }}
+ - name: ELASTIC_APM_SERVER_URL
+ value: {{ .Values.globals.elasticApmServerUrl | quote }}
+ {{ end }}
+ - name: CDN_URL
+ value: {{ .Values.globals.cdnUrl }}
image: budibase/apps:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
@@ -142,7 +170,10 @@ spec:
name: bbapps
ports:
- containerPort: {{ .Values.services.apps.port }}
- resources: {}
+ {{ with .Values.services.apps.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/templates/couchdb-backup.yaml b/charts/budibase/templates/couchdb-backup.yaml
index ae062475ce..68e5eab617 100644
--- a/charts/budibase/templates/couchdb-backup.yaml
+++ b/charts/budibase/templates/couchdb-backup.yaml
@@ -38,7 +38,10 @@ spec:
image: redgeoff/replicate-couchdb-cluster
imagePullPolicy: Always
name: couchdb-backup
- resources: {}
+ {{ with .Values.services.couchdb.backup.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml
index 103f9e3ed2..144dbe539a 100644
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@@ -56,7 +56,10 @@ spec:
name: minio-service
ports:
- containerPort: {{ .Values.services.objectStore.port }}
- resources: {}
+ {{ with .Values.services.objectStore.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
- mountPath: /data
name: minio-data
diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml
index 505a46f1e8..5588022032 100644
--- a/charts/budibase/templates/proxy-service-deployment.yaml
+++ b/charts/budibase/templates/proxy-service-deployment.yaml
@@ -30,7 +30,10 @@ spec:
name: proxy-service
ports:
- containerPort: {{ .Values.services.proxy.port }}
- resources: {}
+ {{ with .Values.services.proxy.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
{{- with .Values.affinity }}
affinity:
diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml
index 6e09346cad..d94e4d70f8 100644
--- a/charts/budibase/templates/redis-service-deployment.yaml
+++ b/charts/budibase/templates/redis-service-deployment.yaml
@@ -35,7 +35,10 @@ spec:
name: redis-service
ports:
- containerPort: {{ .Values.services.redis.port }}
- resources: {}
+ {{ with .Values.services.redis.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
- mountPath: /data
name: redis-data
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 918dab427b..df692a0723 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -27,6 +27,8 @@ spec:
spec:
containers:
- env:
+ - name: BUDIBASE_ENVIRONMENT
+ value: {{ .Values.globals.budibaseEnv }}
- name: DEPLOYMENT_ENVIRONMENT
value: "kubernetes"
- name: CLUSTER_PORT
@@ -75,6 +77,14 @@ spec:
key: objectStoreSecret
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
+ - name: PLUGIN_BUCKET_NAME
+ value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+ - name: APPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.appsBucketName | quote }}
+ - name: GLOBAL_CLOUD_BUCKET_NAME
+ value: {{ .Values.services.objectStore.globalBucketName | quote }}
+ - name: BACKUPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.worker.port | quote }}
- name: MULTI_TENANCY
@@ -125,6 +135,21 @@ spec:
value: {{ .Values.globals.google.secret | quote }}
- name: TENANT_FEATURE_FLAGS
value: {{ .Values.globals.tenantFeatureFlags | quote }}
+ {{ if .Values.globals.elasticApmEnabled }}
+ - name: ELASTIC_APM_ENABLED
+ value: {{ .Values.globals.elasticApmEnabled | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmSecretToken }}
+ - name: ELASTIC_APM_SECRET_TOKEN
+ value: {{ .Values.globals.elasticApmSecretToken | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmServerUrl }}
+ - name: ELASTIC_APM_SERVER_URL
+ value: {{ .Values.globals.elasticApmServerUrl | quote }}
+ {{ end }}
+ - name: CDN_URL
+ value: {{ .Values.globals.cdnUrl }}
+
image: budibase/worker:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
livenessProbe:
@@ -136,7 +161,10 @@ spec:
name: bbworker
ports:
- containerPort: {{ .Values.services.worker.port }}
- resources: {}
+ {{ with .Values.services.worker.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 2734202fff..a2a761aa86 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -60,19 +60,6 @@ ingress:
port:
number: 10000
-resources:
- {}
- # We usually recommend not to specify default resources and to leave this as a conscious
- # choice for the user. This also increases chances charts run on environments with little
- # resources, such as Minikube. If you do want to specify resources, uncomment the following
- # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
-
autoscaling:
enabled: false
minReplicas: 1
@@ -89,9 +76,10 @@ affinity: {}
globals:
appVersion: "latest"
budibaseEnv: PRODUCTION
+ tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS"
enableAnalytics: "1"
sentryDSN: ""
- posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS"
+ posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
logLevel: info
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
@@ -110,10 +98,15 @@ globals:
# if createSecrets is set to false, you can hard-code your secrets here
internalApiKey: ""
jwtSecret: ""
+ cdnUrl: ""
smtp:
enabled: false
+# elasticApmEnabled:
+# elasticApmSecretToken:
+# elasticApmServerUrl:
+
services:
budibaseVersion: latest
dns: cluster.local
@@ -121,15 +114,19 @@ services:
proxy:
port: 10000
replicaCount: 1
+ resources: {}
apps:
port: 4002
replicaCount: 1
logLevel: info
+ resources: {}
+# nodeDebug: "" # set the value of NODE_DEBUG
worker:
port: 4003
replicaCount: 1
+ resources: {}
couchdb:
enabled: true
@@ -143,6 +140,7 @@ services:
target: ""
# backup interval in seconds
interval: ""
+ resources: {}
redis:
enabled: true # disable if using external redis
@@ -156,6 +154,7 @@ services:
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner.
storageClass: ""
+ resources: {}
objectStore:
minio: true
@@ -172,6 +171,7 @@ services:
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner.
storageClass: ""
+ resources: {}
# Override values in couchDB subchart
couchdb:
diff --git a/docs/DEV-SETUP-DEBIAN.md b/docs/DEV-SETUP-DEBIAN.md
index 88a124708c..9edd8286cb 100644
--- a/docs/DEV-SETUP-DEBIAN.md
+++ b/docs/DEV-SETUP-DEBIAN.md
@@ -1,12 +1,15 @@
## Dev Environment on Debian 11
-### Install Node
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
-Budibase requires a recent version of node (14+):
+Install NVM
```
-curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
-apt -y install nodejs
-node -v
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+```
+Install Node 14
+```
+nvm install 14
```
### Install npm requirements
@@ -31,7 +34,7 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show
- Docker: 20.10.5
- Docker-Compose: 1.29.2
-- Node: v16.15.1
+- Node: v14.20.1
- Yarn: 1.22.19
- Lerna: 5.1.4
diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md
index c5990e58da..d9e2dcad6a 100644
--- a/docs/DEV-SETUP-MACOSX.md
+++ b/docs/DEV-SETUP-MACOSX.md
@@ -11,7 +11,7 @@ through brew.
### Install Node
-Budibase requires a recent version of node (14+):
+Budibase requires a recent version of node 14:
```
brew install node npm
node -v
@@ -38,7 +38,7 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show
- Docker: 20.10.14
- Docker-Compose: 2.6.0
-- Node: 18.3.0
+- Node: 14.20.1
- Yarn: 1.22.19
- Lerna: 5.1.4
@@ -59,4 +59,7 @@ The dev version will be available on port 10000 i.e.
http://127.0.0.1:10000/builder/admin
| **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in
-[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
\ No newline at end of file
+[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
+
+### Troubleshooting
+If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11.
diff --git a/docs/DEV-SETUP-WINDOWS.md b/docs/DEV-SETUP-WINDOWS.md
new file mode 100644
index 0000000000..c5608b7567
--- /dev/null
+++ b/docs/DEV-SETUP-WINDOWS.md
@@ -0,0 +1,81 @@
+## Dev Environment on Windows 10/11 (WSL2)
+
+
+### Install WSL with Ubuntu LTS
+
+Enable WSL 2 on Windows 10/11 for docker support.
+```
+wsl --set-default-version 2
+```
+Install Ubuntu LTS.
+```
+wsl --install Ubuntu
+```
+
+Or follow the instruction here:
+https://learn.microsoft.com/en-us/windows/wsl/install
+
+### Install Docker in windows
+Download the installer from docker and install it.
+
+Check this url for more detailed instructions:
+https://docs.docker.com/desktop/install/windows-install/
+
+You should follow the next steps from within the Ubuntu terminal.
+
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
+
+Install NVM
+```
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+```
+Install Node 14
+```
+nvm install 14
+```
+
+
+### Install npm requirements
+
+```
+npm install -g yarn jest lerna
+```
+
+### Clone the repo
+```
+git clone https://github.com/Budibase/budibase.git
+```
+
+### Check Versions
+
+This setup process was tested on Windows 11 with version numbers show below. Your mileage may vary using anything else.
+
+- Docker: 20.10.7
+- Docker-Compose: 2.10.2
+- Node: v14.20.1
+- Yarn: 1.22.19
+- Lerna: 5.5.4
+
+### Build
+
+```
+cd budibase
+yarn setup
+```
+The yarn setup command runs several build steps i.e.
+```
+node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+```
+So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
+
+The dev version will be available on port 10000 i.e.
+
+http://127.0.0.1:10000/builder/admin
+
+### Working with the code
+Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
+
+https://code.visualstudio.com/docs/remote/wsl
+
+Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.
\ No newline at end of file
diff --git a/examples/nextjs-api-sales/definitions/openapi.ts b/examples/nextjs-api-sales/definitions/openapi.ts
index 4f4ad45fc6..7f7f6befec 100644
--- a/examples/nextjs-api-sales/definitions/openapi.ts
+++ b/examples/nextjs-api-sales/definitions/openapi.ts
@@ -348,7 +348,7 @@ export interface paths {
}
}
responses: {
- /** Returns the created table, including the ID which has been generated for it. This can be internal or external data sources. */
+ /** Returns the created table, including the ID which has been generated for it. This can be internal or external datasources. */
200: {
content: {
"application/json": components["schemas"]["tableOutput"]
@@ -959,7 +959,7 @@ export interface components {
query: {
/** @description The ID of the query. */
_id: string
- /** @description The ID of the data source the query belongs to. */
+ /** @description The ID of the datasource the query belongs to. */
datasourceId?: string
/** @description The bindings which are required to perform this query. */
parameters?: string[]
@@ -983,7 +983,7 @@ export interface components {
data: {
/** @description The ID of the query. */
_id: string
- /** @description The ID of the data source the query belongs to. */
+ /** @description The ID of the datasource the query belongs to. */
datasourceId?: string
/** @description The bindings which are required to perform this query. */
parameters?: string[]
diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json
index 6d75c85f01..41ce52e952 100644
--- a/examples/nextjs-api-sales/package.json
+++ b/examples/nextjs-api-sales/package.json
@@ -11,8 +11,8 @@
"dependencies": {
"bulma": "^0.9.3",
"next": "12.1.0",
- "node-fetch": "^3.2.2",
- "node-sass": "^7.0.1",
+ "node-fetch": "^3.2.10",
+ "sass": "^1.52.3",
"react": "17.0.2",
"react-dom": "17.0.2",
"react-notifications-component": "^3.4.1"
@@ -24,4 +24,4 @@
"eslint-config-next": "12.1.0",
"typescript": "4.6.2"
}
-}
+}
\ No newline at end of file
diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock
index 52c89967b2..f47fb84e33 100644
--- a/examples/nextjs-api-sales/yarn.lock
+++ b/examples/nextjs-api-sales/yarn.lock
@@ -2020,10 +2020,10 @@ node-domexception@^1.0.0:
resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
-node-fetch@^3.2.2:
- version "3.2.2"
- resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.2.tgz#16d33fbe32ca7c6ca1ca8ba5dfea1dd885c59f04"
- integrity sha512-Cwhq1JFIoon15wcIkFzubVNFE5GvXGV82pKf4knXXjvGmn7RJKcypeuqcVNZMGDZsAFWyIRya/anwAJr7TWJ7w==
+node-fetch@^3.2.10:
+ version "3.2.10"
+ resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.10.tgz#e8347f94b54ae18b57c9c049ef641cef398a85c8"
+ integrity sha512-MhuzNwdURnZ1Cp4XTazr69K0BTizsBroX7Zx3UgDSVcZYKF/6p0CBe4EUb/hLqmzVhl0UpYfgRljQ4yxE+iCxA==
dependencies:
data-uri-to-buffer "^4.0.0"
fetch-blob "^3.1.4"
diff --git a/hosting/.env b/hosting/.env
index 11dd661bf1..c5638a266f 100644
--- a/hosting/.env
+++ b/hosting/.env
@@ -22,4 +22,7 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
# An admin user can be automatically created initially if these are set
BB_ADMIN_USER_EMAIL=
-BB_ADMIN_USER_PASSWORD=
\ No newline at end of file
+BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=
\ No newline at end of file
diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index f669f9261d..5b2adc2665 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -25,9 +25,12 @@ services:
REDIS_PASSWORD: ${REDIS_PASSWORD}
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+ PLUGINS_DIR: ${PLUGINS_DIR}
depends_on:
- worker-service
- redis-service
+# volumes:
+# - /some/path/to/plugins:/plugins
worker-service:
restart: unless-stopped
@@ -76,6 +79,9 @@ services:
- "${MAIN_PORT}:10000"
container_name: bbproxy
image: budibase/proxy
+ environment:
+ - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ - PROXY_RATE_LIMIT_API_PER_SECOND=20
depends_on:
- minio-service
- worker-service
diff --git a/hosting/hosting.properties b/hosting/hosting.properties
index 11dd661bf1..c5638a266f 100644
--- a/hosting/hosting.properties
+++ b/hosting/hosting.properties
@@ -22,4 +22,7 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
# An admin user can be automatically created initially if these are set
BB_ADMIN_USER_EMAIL=
-BB_ADMIN_USER_PASSWORD=
\ No newline at end of file
+BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=
\ No newline at end of file
diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs
index 9398b7e719..93a07435e5 100644
--- a/hosting/nginx.dev.conf.hbs
+++ b/hosting/nginx.dev.conf.hbs
@@ -15,12 +15,30 @@ http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
+ '"$http_user_agent" "$http_x_forwarded_for" '
+ 'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+ access_log /var/log/nginx/access.log main;
map $http_upgrade $connection_upgrade {
default "upgrade";
}
+ upstream app-service {
+ server {{address}}:4001;
+ keepalive 32;
+ }
+
+ upstream worker-service {
+ server {{address}}:4002;
+ keepalive 32;
+ }
+
+ upstream builder {
+ server {{address}}:3000;
+ keepalive 32;
+ }
+
server {
listen 10000 default_server;
server_name _;
@@ -40,41 +58,118 @@ http {
}
location ~ ^/api/(system|admin|global)/ {
- proxy_pass http://{{ address }}:4002;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://worker-service;
+ }
+
+ location /api/backups/ {
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+ proxy_pass http://app-service;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
}
location /api/ {
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
- proxy_send_timeout 120s;
- proxy_pass http://{{ address }}:4001;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location = / {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /app_ {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /app {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /builder {
- proxy_pass http://{{ address }}:3000;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://builder;
rewrite ^/builder(.*)$ /builder/$1 break;
}
location /builder/ {
- proxy_pass http://{{ address }}:3000;
-
proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+
+ proxy_pass http://builder;
+ }
+
+ location /vite/ {
+ proxy_pass http://builder;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ rewrite ^/vite(.*)$ /$1 break;
+ }
+
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://app-service;
}
location / {
diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs
index ac35a2020d..6f0f1b420d 100644
--- a/hosting/nginx.prod.conf.hbs
+++ b/hosting/nginx.prod.conf.hbs
@@ -9,7 +9,11 @@ events {
}
http {
- limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+ # rate limiting
+ limit_req_status 429;
+ limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=${PROXY_RATE_LIMIT_API_PER_SECOND}r/s;
+ limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
+
include /etc/nginx/mime.types;
default_type application/octet-stream;
proxy_set_header Host $host;
@@ -29,7 +33,10 @@ http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
+ '"$http_user_agent" "$http_x_forwarded_for" '
+ 'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+ access_log /var/log/nginx/access.log main;
map $http_upgrade $connection_upgrade {
default "upgrade";
@@ -44,11 +51,11 @@ http {
proxy_buffering off;
set $csp_default "default-src 'self'";
- set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
+ set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
- set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
+ set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
@@ -90,24 +97,50 @@ http {
proxy_pass http://$watchtower:8080;
}
{{/if}}
+
location ~ ^/(builder|app_) {
proxy_http_version 1.1;
+
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
+
proxy_pass http://$apps:4002;
}
location ~ ^/api/(system|admin|global)/ {
+ proxy_set_header Host $host;
+
proxy_pass http://$worker:4003;
}
location /worker/ {
+ proxy_set_header Host $host;
+
proxy_pass http://$worker:4003;
rewrite ^/worker/(.*)$ /$1 break;
}
+ location /api/backups/ {
+ # calls to export apps are limited
+ limit_req zone=ratelimit burst=20 nodelay;
+
+ # 1800s timeout for app export requests
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_pass http://$apps:4002;
+ }
+
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=20 nodelay;
@@ -122,6 +155,27 @@ http {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
+
+ proxy_pass http://$apps:4002;
+ }
+
+ location /api/webhooks/ {
+ # calls to webhooks are rate limited
+ limit_req zone=webhooks nodelay;
+
+ # Rest of configuration copied from /api/ location above
+ # 120s timeout on API requests
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
proxy_pass http://$apps:4002;
}
@@ -131,15 +185,26 @@ http {
rewrite ^/db/(.*)$ /$1 break;
}
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://$apps:4002;
+ }
+
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Host $http_host;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
+
proxy_pass http://$minio:9000;
}
diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh
new file mode 100644
index 0000000000..e2e89388a9
--- /dev/null
+++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+ # ipv6 not available so delete lines from nginx conf
+ if [ -f "$NGINX_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $NGINX_CONF_FILE
+ fi
+ if [ -f "$DEFAULT_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE
+ fi
+ echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+ echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0
diff --git a/hosting/proxy/80-listen-on-ipv6-by-default.sh b/hosting/proxy/80-listen-on-ipv6-by-default.sh
new file mode 100644
index 0000000000..1d62732ea1
--- /dev/null
+++ b/hosting/proxy/80-listen-on-ipv6-by-default.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+ # ipv6 not available so delete lines from nginx conf
+ if [ -f "$NGINX_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $NGINX_CONF_FILE
+ fi
+ if [ -f "$DEFAULT_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE
+ fi
+ echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+ echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0
\ No newline at end of file
diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile
index a2b17d3333..68e8134750 100644
--- a/hosting/proxy/Dockerfile
+++ b/hosting/proxy/Dockerfile
@@ -1,3 +1,19 @@
FROM nginx:latest
-COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
-COPY error.html /usr/share/nginx/html/error.html
\ No newline at end of file
+
+# nginx.conf
+# use the default nginx behaviour for *.template files which are processed with envsubst
+# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
+ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+
+# IPv6 removal needs to happen after envsubst
+RUN rm -rf /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
+COPY 80-listen-on-ipv6-by-default.sh /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh
+RUN chmod +x /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh
+
+# Error handling
+COPY error.html /usr/share/nginx/html/error.html
+
+# Default environment
+ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
\ No newline at end of file
diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh
index 4c165d12e7..67e1765ca8 100644
--- a/hosting/scripts/build-target-paths.sh
+++ b/hosting/scripts/build-target-paths.sh
@@ -3,15 +3,22 @@
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persisent data & SSH on port 2222
- mkdir -p /home/{search,minio,couch}
- mkdir -p /home/couch/{dbs,views}
- chown -R couchdb:couchdb /home/couch/
+ DATA_DIR=/home
+ WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+ mkdir -p $DATA_DIR/{search,minio,couch}
+ mkdir -p $DATA_DIR/couch/{dbs,views}
+ chown -R couchdb:couchdb $DATA_DIR/couch/
apt update
apt-get install -y openssh-server
- sed -i 's#dir=/opt/couchdb/data/search#dir=/home/search#' /opt/clouseau/clouseau.ini
- sed -i 's#/minio/minio server /minio &#/minio/minio server /home/minio &#' /runner.sh
- sed -i 's#database_dir = ./data#database_dir = /home/couch/dbs#' /opt/couchdb/etc/default.ini
- sed -i 's#view_index_dir = ./data#view_index_dir = /home/couch/views#' /opt/couchdb/etc/default.ini
- sed -i "s/#Port 22/Port 2222/" /etc/ssh/sshd_config
+ echo "root:Docker!" | chpasswd
+ mkdir -p /tmp
+ chmod +x /tmp/ssh_setup.sh \
+ && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+ cp /etc/sshd_config /etc/ssh/sshd_config
/etc/init.d/ssh restart
-fi
+ sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+ sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+else
+ sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+ sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
\ No newline at end of file
diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index b5bf17adde..58796f0362 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -19,32 +19,18 @@ ADD packages/worker .
RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
FROM couchdb:3.2.1
-# TARGETARCH can be amd64 or arm e.g. docker build --build-arg TARGETARCH=amd64
-ARG TARGETARCH amd64
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
# e.g. docker build --build-arg TARGETBUILD=aas ....
-ARG TARGETBUILD single
+ARG TARGETBUILD=single
ENV TARGETBUILD $TARGETBUILD
COPY --from=build /app /app
COPY --from=build /worker /worker
-ENV \
- APP_PORT=4001 \
- ARCHITECTURE=amd \
- BUDIBASE_ENVIRONMENT=PRODUCTION \
- CLUSTER_PORT=80 \
- # CUSTOM_DOMAIN=budi001.custom.com \
- DEPLOYMENT_ENVIRONMENT=docker \
- MINIO_URL=http://localhost:9000 \
- POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \
- REDIS_URL=localhost:6379 \
- SELF_HOSTED=1 \
- TARGETBUILD=$TARGETBUILD \
- WORKER_PORT=4002 \
- WORKER_URL=http://localhost:4002 \
- APPS_URL=http://localhost:4001
-
+# ENV CUSTOM_DOMAIN=budi001.custom.com \
+# See runner.sh for Env Vars
# These secret env variables are generated by the runner at startup
# their values can be overriden by the user, they will be written
# to the .env file in the /data directory for use later on
@@ -114,7 +100,10 @@ RUN chmod +x ./healthcheck.sh
ADD hosting/scripts/build-target-paths.sh .
RUN chmod +x ./build-target-paths.sh
+# Script below sets the path for storing data based on $DATA_DIR
# For Azure App Service install SSH & point data locations to /home
+ADD hosting/single/ssh/sshd_config /etc/
+ADD hosting/single/ssh/ssh_setup.sh /tmp
RUN /build-target-paths.sh
# cleanup cache
@@ -122,6 +111,8 @@ RUN yarn cache clean -f
EXPOSE 80
EXPOSE 443
+# Expose port 2222 for SSH on Azure App Service build
+EXPOSE 2222
VOLUME /data
# setup letsencrypt certificate
diff --git a/hosting/single/clouseau/clouseau.ini b/hosting/single/clouseau/clouseau.ini
index 78e43744e5..578a5acafa 100644
--- a/hosting/single/clouseau/clouseau.ini
+++ b/hosting/single/clouseau/clouseau.ini
@@ -7,7 +7,7 @@ name=clouseau@127.0.0.1
cookie=monster
; the path where you would like to store the search index files
-dir=/data/search
+dir=DATA_DIR/search
; the number of search indexes that can be open simultaneously
max_indexes_open=500
diff --git a/hosting/single/couch/local.ini b/hosting/single/couch/local.ini
index 72872a60e1..266c0d4b60 100644
--- a/hosting/single/couch/local.ini
+++ b/hosting/single/couch/local.ini
@@ -1,5 +1,5 @@
; CouchDB Configuration Settings
[couchdb]
-database_dir = /data/couch/dbs
-view_index_dir = /data/couch/views
+database_dir = DATA_DIR/couch/dbs
+view_index_dir = DATA_DIR/couch/views
diff --git a/hosting/single/healthcheck.sh b/hosting/single/healthcheck.sh
index b92cd153a3..592b3e94fa 100644
--- a/hosting/single/healthcheck.sh
+++ b/hosting/single/healthcheck.sh
@@ -3,6 +3,11 @@ healthy=true
if [ -f "/data/.env" ]; then
export $(cat /data/.env | xargs)
+elif [ -f "/home/.env" ]; then
+ export $(cat /home/.env | xargs)
+else
+ echo "No .env file found"
+ healthy=false
fi
if [[ $(curl -Lfk -s -w "%{http_code}\n" http://localhost/ -o /dev/null) -ne 200 ]]; then
diff --git a/hosting/single/nginx/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf
index c0d80a0185..9a5ec91c1f 100644
--- a/hosting/single/nginx/nginx-default-site.conf
+++ b/hosting/single/nginx/nginx-default-site.conf
@@ -43,6 +43,24 @@ server {
rewrite ^/worker/(.*)$ /$1 break;
}
+ location /api/backups/ {
+ # calls to export apps are limited
+ limit_req zone=ratelimit burst=20 nodelay;
+
+ # 1800s timeout for app export requests
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_pass http://127.0.0.1:4001;
+ }
+
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=20 nodelay;
@@ -66,6 +84,15 @@ server {
rewrite ^/db/(.*)$ /$1 break;
}
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://127.0.0.1:4001;
+ }
+
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index 9abb2fd093..ea825131db 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -1,9 +1,47 @@
#!/bin/bash
-declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
-if [ -f "/data/.env" ]; then
- export $(cat /data/.env | xargs)
+declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
+declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL")
+# Check the env vars set in Dockerfile have come through, AAS seems to drop them
+[[ -z "${APP_PORT}" ]] && export APP_PORT=4001
+[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd
+[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
+[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
+[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
+[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
+[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
+[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS"
+[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
+[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
+[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
+[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
+[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002
+[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001
+# export CUSTOM_DOMAIN=budi001.custom.com
+
+# Azure App Service customisations
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+ DATA_DIR=/home
+ WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+ /etc/init.d/ssh start
+else
+ DATA_DIR=${DATA_DIR:-/data}
fi
-# first randomise any unset environment variables
+mkdir -p ${DATA_DIR}
+# Mount NFS or GCP Filestore if env vars exist for it
+if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+ echo "Mounting NFS share"
+ apt update && apt install -y nfs-common nfs-kernel-server
+ echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
+ mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
+ echo "Mounting result: $?"
+fi
+
+if [ -f "${DATA_DIR}/.env" ]; then
+ # Read in the .env file and export the variables
+ for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+fi
+# randomise any unset environment variables
for ENV_VAR in "${ENV_VARS[@]}"
do
temp=$(eval "echo \$$ENV_VAR")
@@ -14,21 +52,33 @@ done
if [[ -z "${COUCH_DB_URL}" ]]; then
export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984
fi
-if [ ! -f "/data/.env" ]; then
- touch /data/.env
+if [ ! -f "${DATA_DIR}/.env" ]; then
+ touch ${DATA_DIR}/.env
for ENV_VAR in "${ENV_VARS[@]}"
do
temp=$(eval "echo \$$ENV_VAR")
- echo "$ENV_VAR=$temp" >> /data/.env
+ echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
done
+ for ENV_VAR in "${DOCKER_VARS[@]}"
+ do
+ temp=$(eval "echo \$$ENV_VAR")
+ echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+ done
+ echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env
fi
+# Read in the .env file and export the variables
+for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+ln -s ${DATA_DIR}/.env /app/.env
+ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
-mkdir -p /data/couch/{dbs,views} /home/couch/{dbs,views}
-chown -R couchdb:couchdb /data/couch /home/couch
-redis-server --requirepass $REDIS_PASSWORD &
-/opt/clouseau/bin/clouseau &
-/minio/minio server /data/minio &
+mkdir -p ${DATA_DIR}/couch/{dbs,views}
+mkdir -p ${DATA_DIR}/minio
+mkdir -p ${DATA_DIR}/search
+chown -R couchdb:couchdb ${DATA_DIR}/couch
+redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
/etc/init.d/nginx restart
if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
@@ -37,16 +87,18 @@ if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
chmod +x /etc/cron.d/certificate-renew
# Request the certbot certificate
/app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN}
+ /etc/init.d/nginx restart
fi
-/etc/init.d/nginx restart
pushd app
-pm2 start --name app "yarn run:docker"
+pm2 start -l /dev/stdout --name app "yarn run:docker"
popd
pushd worker
-pm2 start --name worker "yarn run:docker"
+pm2 start -l /dev/stdout --name worker "yarn run:docker"
popd
sleep 10
+echo "curl to couchdb endpoints"
curl -X PUT ${COUCH_DB_URL}/_users
curl -X PUT ${COUCH_DB_URL}/_replicator
+echo "end of runner.sh, sleeping ..."
sleep infinity
diff --git a/hosting/single/ssh/ssh_setup.sh b/hosting/single/ssh/ssh_setup.sh
new file mode 100644
index 0000000000..0af0b6d7ad
--- /dev/null
+++ b/hosting/single/ssh/ssh_setup.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+ssh-keygen -A
+
+#prepare run dir
+if [ ! -d "/var/run/sshd" ]; then
+ mkdir -p /var/run/sshd
+fi
\ No newline at end of file
diff --git a/hosting/single/ssh/sshd_config b/hosting/single/ssh/sshd_config
new file mode 100644
index 0000000000..7eb5df953a
--- /dev/null
+++ b/hosting/single/ssh/sshd_config
@@ -0,0 +1,12 @@
+Port 2222
+ListenAddress 0.0.0.0
+LoginGraceTime 180
+X11Forwarding yes
+Ciphers aes128-cbc,3des-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr
+MACs hmac-sha1,hmac-sha1-96
+StrictModes yes
+SyslogFacility DAEMON
+PasswordAuthentication yes
+PermitEmptyPasswords no
+PermitRootLogin yes
+Subsystem sftp internal-sftp
diff --git a/lerna.json b/lerna.json
index 6394390f18..535393e618 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
{
- "version": "1.2.20",
+ "version": "2.1.40-alpha.0",
"npmClient": "yarn",
"packages": [
"packages/*"
diff --git a/package.json b/package.json
index 4c24e0025b..af513fc8dd 100644
--- a/package.json
+++ b/package.json
@@ -3,7 +3,6 @@
"private": true,
"devDependencies": {
"@rollup/plugin-json": "^4.0.2",
- "@types/mongodb": "3.6.3",
"@typescript-eslint/parser": "4.28.0",
"babel-eslint": "^10.0.3",
"eslint": "^7.28.0",
@@ -13,18 +12,21 @@
"js-yaml": "^4.1.0",
"kill-port": "^1.6.1",
"lerna": "3.14.1",
+ "madge": "^5.0.1",
"prettier": "^2.3.1",
"prettier-plugin-svelte": "^2.3.0",
"rimraf": "^3.0.2",
"rollup-plugin-replace": "^2.2.0",
"svelte": "^3.38.2",
- "typescript": "4.5.5"
+ "typescript": "4.7.3"
},
"scripts": {
"setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
"bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
"build": "lerna run build",
"build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+ "build:sdk": "lerna run build:sdk",
+ "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
"release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
"release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop",
"release:pro": "bash scripts/pro/release.sh",
@@ -45,8 +47,8 @@
"lint:eslint": "eslint packages",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
"lint": "yarn run lint:eslint && yarn run lint:prettier",
- "lint:fix:eslint": "eslint --fix packages",
- "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
+ "lint:fix:eslint": "eslint --fix packages qa-core",
+ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
"lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
"test:e2e": "lerna run cy:test --stream",
"test:e2e:ci": "lerna run cy:ci --stream",
@@ -73,8 +75,8 @@
"env:multi:disable": "lerna run env:multi:disable",
"env:selfhost:enable": "lerna run env:selfhost:enable",
"env:selfhost:disable": "lerna run env:selfhost:disable",
- "env:localdomain:enable": "lerna run env:localdomain:enable",
- "env:localdomain:disable": "lerna run env:localdomain:disable",
+ "env:localdomain:enable": "./scripts/localdomain.sh enable",
+ "env:localdomain:disable": "./scripts/localdomain.sh disable",
"env:account:enable": "lerna run env:account:enable",
"env:account:disable": "lerna run env:account:disable",
"mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable",
diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts
new file mode 100644
index 0000000000..7fac80faa9
--- /dev/null
+++ b/packages/backend-core/__mocks__/aws-sdk.ts
@@ -0,0 +1,15 @@
+const mockS3 = {
+ headBucket: jest.fn().mockReturnThis(),
+ deleteObject: jest.fn().mockReturnThis(),
+ deleteObjects: jest.fn().mockReturnThis(),
+ createBucket: jest.fn().mockReturnThis(),
+ listObjects: jest.fn().mockReturnThis(),
+ promise: jest.fn().mockReturnThis(),
+ catch: jest.fn(),
+}
+
+const AWS = {
+ S3: jest.fn(() => mockS3),
+}
+
+export default AWS
diff --git a/packages/backend-core/accounts.js b/packages/backend-core/accounts.js
deleted file mode 100644
index 47ad03456a..0000000000
--- a/packages/backend-core/accounts.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/cloud/accounts")
diff --git a/packages/backend-core/auth.js b/packages/backend-core/auth.js
deleted file mode 100644
index bbfe3d41dd..0000000000
--- a/packages/backend-core/auth.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/auth")
diff --git a/packages/backend-core/cache.js b/packages/backend-core/cache.js
deleted file mode 100644
index c8bd3c9b6f..0000000000
--- a/packages/backend-core/cache.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const generic = require("./src/cache/generic")
-
-module.exports = {
- user: require("./src/cache/user"),
- app: require("./src/cache/appMetadata"),
- writethrough: require("./src/cache/writethrough"),
- ...generic,
- cache: generic,
-}
diff --git a/packages/backend-core/constants.js b/packages/backend-core/constants.js
deleted file mode 100644
index 4abb7703db..0000000000
--- a/packages/backend-core/constants.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/constants")
diff --git a/packages/backend-core/context.js b/packages/backend-core/context.js
deleted file mode 100644
index aaa0f56f92..0000000000
--- a/packages/backend-core/context.js
+++ /dev/null
@@ -1,22 +0,0 @@
-const {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
-} = require("./src/context")
-
-const identity = require("./src/context/identity")
-
-module.exports = {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
- identity,
-}
diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js
deleted file mode 100644
index 0d2869d9f1..0000000000
--- a/packages/backend-core/db.js
+++ /dev/null
@@ -1,7 +0,0 @@
-module.exports = {
- ...require("./src/db/utils"),
- ...require("./src/db/constants"),
- ...require("./src/db"),
- ...require("./src/db/views"),
- ...require("./src/db/pouch"),
-}
diff --git a/packages/backend-core/deprovision.js b/packages/backend-core/deprovision.js
deleted file mode 100644
index 672da214ff..0000000000
--- a/packages/backend-core/deprovision.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/context/deprovision")
diff --git a/packages/backend-core/encryption.js b/packages/backend-core/encryption.js
deleted file mode 100644
index 4ccb6e3a99..0000000000
--- a/packages/backend-core/encryption.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/security/encryption")
diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts
new file mode 100644
index 0000000000..d0e5d3d4e7
--- /dev/null
+++ b/packages/backend-core/jest.config.ts
@@ -0,0 +1,21 @@
+import { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+ preset: "ts-jest",
+ testEnvironment: "node",
+ setupFiles: ["./tests/jestSetup.ts"],
+ collectCoverageFrom: ["src/**/*.{js,ts}"],
+ coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+ // use sources when not in CI
+ config.moduleNameMapper = {
+ "@budibase/types": "/../types/src",
+ "^axios.*$": "/node_modules/axios/lib/axios.js",
+ }
+} else {
+ console.log("Running tests with compiled dependency sources")
+}
+
+export default config
diff --git a/packages/backend-core/logging.js b/packages/backend-core/logging.js
deleted file mode 100644
index da40fe3100..0000000000
--- a/packages/backend-core/logging.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/logging")
diff --git a/packages/backend-core/middleware.js b/packages/backend-core/middleware.js
deleted file mode 100644
index 30fec96239..0000000000
--- a/packages/backend-core/middleware.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/middleware")
diff --git a/packages/backend-core/migrations.js b/packages/backend-core/migrations.js
deleted file mode 100644
index 2de19ebf65..0000000000
--- a/packages/backend-core/migrations.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/migrations")
diff --git a/packages/backend-core/objectStore.js b/packages/backend-core/objectStore.js
deleted file mode 100644
index 3ee433f224..0000000000
--- a/packages/backend-core/objectStore.js
+++ /dev/null
@@ -1,4 +0,0 @@
-module.exports = {
- ...require("./src/objectStore"),
- ...require("./src/objectStore/utils"),
-}
diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 0bc098a783..6abd448ce7 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
- "version": "1.2.20",
+ "version": "2.1.40-alpha.0",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@@ -16,23 +16,27 @@
"prepack": "cp package.json dist",
"build": "tsc -p tsconfig.build.json",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
- "test": "jest",
+ "test": "jest --coverage",
"test:watch": "jest --watchAll"
},
"dependencies": {
- "@budibase/types": "^1.2.20",
+ "@budibase/types": "2.1.40-alpha.0",
+ "@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-sdk": "2.1030.0",
"bcrypt": "5.0.1",
+ "bcryptjs": "2.4.3",
+ "bull": "4.10.1",
"dotenv": "16.0.1",
"emitter-listener": "1.1.2",
"ioredis": "4.28.0",
+ "joi": "17.6.0",
"jsonwebtoken": "8.5.1",
"koa-passport": "4.1.4",
"lodash": "4.17.21",
"lodash.isarguments": "3.1.0",
+ "nano": "^10.1.0",
"node-fetch": "2.6.7",
- "passport-google-auth": "1.0.2",
"passport-google-oauth": "2.0.0",
"passport-jwt": "4.0.0",
"passport-local": "1.0.0",
@@ -48,20 +52,11 @@
"uuid": "8.3.2",
"zlib": "1.0.5"
},
- "jest": {
- "preset": "ts-jest",
- "testEnvironment": "node",
- "moduleNameMapper": {
- "@budibase/types": "/../types/src"
- },
- "setupFiles": [
- "./scripts/jestSetup.ts"
- ]
- },
"devDependencies": {
- "@shopify/jest-koa-mocks": "3.1.5",
+ "@types/chance": "1.1.3",
+ "@types/ioredis": "4.28.0",
"@types/jest": "27.5.1",
- "@types/koa": "2.0.52",
+ "@types/koa": "2.13.4",
"@types/lodash": "4.14.180",
"@types/node": "14.18.20",
"@types/node-fetch": "2.6.1",
@@ -70,13 +65,16 @@
"@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1",
"@types/uuid": "8.3.4",
+ "chance": "1.1.3",
"ioredis-mock": "5.8.0",
- "jest": "27.5.1",
- "koa": "2.7.0",
+ "jest": "28.1.1",
+ "koa": "2.13.4",
"nodemon": "2.0.16",
"pouchdb-adapter-memory": "7.2.2",
"timekeeper": "2.2.0",
- "ts-jest": "27.1.5",
+ "ts-jest": "28.0.4",
+ "ts-node": "10.8.1",
+ "tsconfig-paths": "4.0.0",
"typescript": "4.7.3"
},
"gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
diff --git a/packages/backend-core/permissions.js b/packages/backend-core/permissions.js
deleted file mode 100644
index 42f37c9c7e..0000000000
--- a/packages/backend-core/permissions.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/security/permissions")
diff --git a/packages/backend-core/redis.js b/packages/backend-core/redis.js
deleted file mode 100644
index 1f7a48540a..0000000000
--- a/packages/backend-core/redis.js
+++ /dev/null
@@ -1,5 +0,0 @@
-module.exports = {
- Client: require("./src/redis"),
- utils: require("./src/redis/utils"),
- clients: require("./src/redis/init"),
-}
diff --git a/packages/backend-core/roles.js b/packages/backend-core/roles.js
deleted file mode 100644
index 158bcdb6b8..0000000000
--- a/packages/backend-core/roles.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/security/roles")
diff --git a/packages/backend-core/scripts/jestSetup.ts b/packages/backend-core/scripts/jestSetup.ts
deleted file mode 100644
index 1e86649a24..0000000000
--- a/packages/backend-core/scripts/jestSetup.ts
+++ /dev/null
@@ -1,12 +0,0 @@
-import env from "../src/environment"
-import { mocks } from "../tests/utilities"
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")
diff --git a/packages/backend-core/sessions.js b/packages/backend-core/sessions.js
deleted file mode 100644
index c07efa2380..0000000000
--- a/packages/backend-core/sessions.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/security/sessions")
diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth/auth.ts
similarity index 60%
rename from packages/backend-core/src/auth.js
rename to packages/backend-core/src/auth/auth.ts
index d39b8426fb..75e425bd0f 100644
--- a/packages/backend-core/src/auth.js
+++ b/packages/backend-core/src/auth/auth.ts
@@ -1,19 +1,16 @@
-const passport = require("koa-passport")
+const _passport = require("koa-passport")
const LocalStrategy = require("passport-local").Strategy
const JwtStrategy = require("passport-jwt").Strategy
-const { getGlobalDB } = require("./tenancy")
+import { getGlobalDB } from "../tenancy"
const refresh = require("passport-oauth2-refresh")
-const { Configs } = require("./constants")
-const { getScopedConfig } = require("./db/utils")
-const {
- jwt,
+import { Config } from "../constants"
+import { getScopedConfig } from "../db"
+import {
+ jwt as jwtPassport,
local,
authenticated,
- google,
- oidc,
auditLog,
tenancy,
- appTenancy,
authError,
ssoCallbackUrl,
csrf,
@@ -22,32 +19,60 @@ const {
builderOnly,
builderOrAdmin,
joiValidator,
-} = require("./middleware")
-
-const { invalidateUser } = require("./cache/user")
+ oidc,
+ google,
+} from "../middleware"
+import { invalidateUser } from "../cache/user"
+import { User } from "@budibase/types"
+import { logAlert } from "../logging"
+export {
+ auditLog,
+ authError,
+ internalApi,
+ ssoCallbackUrl,
+ adminOnly,
+ builderOnly,
+ builderOrAdmin,
+ joiValidator,
+ google,
+ oidc,
+} from "../middleware"
+export const buildAuthMiddleware = authenticated
+export const buildTenancyMiddleware = tenancy
+export const buildCsrfMiddleware = csrf
+export const passport = _passport
+export const jwt = require("jsonwebtoken")
// Strategies
-passport.use(new LocalStrategy(local.options, local.authenticate))
-passport.use(new JwtStrategy(jwt.options, jwt.authenticate))
+_passport.use(new LocalStrategy(local.options, local.authenticate))
+if (jwtPassport.options.secretOrKey) {
+ _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate))
+} else {
+ logAlert("No JWT Secret supplied, cannot configure JWT strategy")
+}
-passport.serializeUser((user, done) => done(null, user))
+_passport.serializeUser((user: User, done: any) => done(null, user))
-passport.deserializeUser(async (user, done) => {
+_passport.deserializeUser(async (user: User, done: any) => {
const db = getGlobalDB()
try {
- const user = await db.get(user._id)
- return done(null, user)
+ const dbUser = await db.get(user._id)
+ return done(null, dbUser)
} catch (err) {
console.error(`User not found`, err)
return done(null, false, { message: "User not found" })
}
})
-async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) {
+async function refreshOIDCAccessToken(
+ db: any,
+ chosenConfig: any,
+ refreshToken: string
+) {
const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig)
- let enrichedConfig
- let strategy
+ let enrichedConfig: any
+ let strategy: any
try {
enrichedConfig = await oidc.fetchStrategyConfig(chosenConfig, callbackUrl)
@@ -68,40 +93,50 @@ async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) {
return new Promise(resolve => {
refresh.requestNewAccessToken(
- Configs.OIDC,
+ Config.OIDC,
refreshToken,
- (err, accessToken, refreshToken, params) => {
+ (err: any, accessToken: string, refreshToken: any, params: any) => {
resolve({ err, accessToken, refreshToken, params })
}
)
})
}
-async function refreshGoogleAccessToken(db, config, refreshToken) {
+async function refreshGoogleAccessToken(
+ db: any,
+ config: any,
+ refreshToken: any
+) {
let callbackUrl = await google.getCallbackUrl(db, config)
let strategy
try {
strategy = await google.strategyFactory(config, callbackUrl)
- } catch (err) {
+ } catch (err: any) {
console.error(err)
- throw new Error("Error constructing OIDC refresh strategy", err)
+ throw new Error(
+ `Error constructing OIDC refresh strategy: message=${err.message}`
+ )
}
refresh.use(strategy)
return new Promise(resolve => {
refresh.requestNewAccessToken(
- Configs.GOOGLE,
+ Config.GOOGLE,
refreshToken,
- (err, accessToken, refreshToken, params) => {
+ (err: any, accessToken: string, refreshToken: string, params: any) => {
resolve({ err, accessToken, refreshToken, params })
}
)
})
}
-async function refreshOAuthToken(refreshToken, configType, configId) {
+export async function refreshOAuthToken(
+ refreshToken: string,
+ configType: string,
+ configId: string
+) {
const db = getGlobalDB()
const config = await getScopedConfig(db, {
@@ -111,9 +146,9 @@ async function refreshOAuthToken(refreshToken, configType, configId) {
let chosenConfig = {}
let refreshResponse
- if (configType === Configs.OIDC) {
+ if (configType === Config.OIDC) {
// configId - retrieved from cookie.
- chosenConfig = config.configs.filter(c => c.uuid === configId)[0]
+ chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
if (!chosenConfig) {
throw new Error("Invalid OIDC configuration")
}
@@ -134,7 +169,7 @@ async function refreshOAuthToken(refreshToken, configType, configId) {
return refreshResponse
}
-async function updateUserOAuth(userId, oAuthConfig) {
+export async function updateUserOAuth(userId: string, oAuthConfig: any) {
const details = {
accessToken: oAuthConfig.accessToken,
refreshToken: oAuthConfig.refreshToken,
@@ -161,24 +196,3 @@ async function updateUserOAuth(userId, oAuthConfig) {
console.error("Could not update OAuth details for current user", e)
}
}
-
-module.exports = {
- buildAuthMiddleware: authenticated,
- passport,
- google,
- oidc,
- jwt: require("jsonwebtoken"),
- buildTenancyMiddleware: tenancy,
- buildAppTenancyMiddleware: appTenancy,
- auditLog,
- authError,
- buildCsrfMiddleware: csrf,
- internalApi,
- refreshOAuthToken,
- updateUserOAuth,
- ssoCallbackUrl,
- adminOnly,
- builderOnly,
- builderOrAdmin,
- joiValidator,
-}
diff --git a/packages/backend-core/src/auth/index.ts b/packages/backend-core/src/auth/index.ts
new file mode 100644
index 0000000000..306751af96
--- /dev/null
+++ b/packages/backend-core/src/auth/index.ts
@@ -0,0 +1 @@
+export * from "./auth"
diff --git a/packages/backend-core/src/cache/appMetadata.js b/packages/backend-core/src/cache/appMetadata.ts
similarity index 78%
rename from packages/backend-core/src/cache/appMetadata.js
rename to packages/backend-core/src/cache/appMetadata.ts
index b0d9481cbd..d24c4a3140 100644
--- a/packages/backend-core/src/cache/appMetadata.js
+++ b/packages/backend-core/src/cache/appMetadata.ts
@@ -1,6 +1,6 @@
-const redis = require("../redis/init")
-const { doWithDB } = require("../db")
-const { DocumentTypes } = require("../db/constants")
+import { getAppClient } from "../redis/init"
+import { doWithDB, DocumentType } from "../db"
+import { Database } from "@budibase/types"
const AppState = {
INVALID: "invalid",
@@ -10,17 +10,17 @@ const EXPIRY_SECONDS = 3600
/**
* The default populate app metadata function
*/
-const populateFromDB = async appId => {
+async function populateFromDB(appId: string) {
return doWithDB(
appId,
- db => {
- return db.get(DocumentTypes.APP_METADATA)
+ (db: Database) => {
+ return db.get(DocumentType.APP_METADATA)
},
{ skip_setup: true }
)
}
-const isInvalid = metadata => {
+function isInvalid(metadata?: { state: string }) {
return !metadata || metadata.state === AppState.INVALID
}
@@ -31,15 +31,15 @@ const isInvalid = metadata => {
* @param {string} appId the id of the app to get metadata from.
* @returns {object} the app metadata.
*/
-exports.getAppMetadata = async appId => {
- const client = await redis.getAppClient()
+export async function getAppMetadata(appId: string) {
+ const client = await getAppClient()
// try cache
let metadata = await client.get(appId)
if (!metadata) {
- let expiry = EXPIRY_SECONDS
+ let expiry: number | undefined = EXPIRY_SECONDS
try {
metadata = await populateFromDB(appId)
- } catch (err) {
+ } catch (err: any) {
// app DB left around, but no metadata, it is invalid
if (err && err.status === 404) {
metadata = { state: AppState.INVALID }
@@ -74,11 +74,11 @@ exports.getAppMetadata = async appId => {
* @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with.
* @return {Promise} will respond with success when cache is updated.
*/
-exports.invalidateAppMetadata = async (appId, newMetadata = null) => {
+export async function invalidateAppMetadata(appId: string, newMetadata?: any) {
if (!appId) {
throw "Cannot invalidate if no app ID provided."
}
- const client = await redis.getAppClient()
+ const client = await getAppClient()
await client.delete(appId)
if (newMetadata) {
await client.store(appId, newMetadata, EXPIRY_SECONDS)
diff --git a/packages/backend-core/src/cache/base/index.ts b/packages/backend-core/src/cache/base/index.ts
index f3216531f4..ab620a900e 100644
--- a/packages/backend-core/src/cache/base/index.ts
+++ b/packages/backend-core/src/cache/base/index.ts
@@ -1,6 +1,6 @@
import { getTenantId } from "../../context"
-import redis from "../../redis/init"
-import RedisWrapper from "../../redis"
+import * as redis from "../../redis/init"
+import { Client } from "../../redis"
function generateTenantKey(key: string) {
const tenantId = getTenantId()
@@ -8,9 +8,9 @@ function generateTenantKey(key: string) {
}
export = class BaseCache {
- client: RedisWrapper | undefined
+ client: Client | undefined
- constructor(client: RedisWrapper | undefined = undefined) {
+ constructor(client: Client | undefined = undefined) {
this.client = client
}
diff --git a/packages/backend-core/src/cache/generic.js b/packages/backend-core/src/cache/generic.js
deleted file mode 100644
index e2f3915339..0000000000
--- a/packages/backend-core/src/cache/generic.js
+++ /dev/null
@@ -1,29 +0,0 @@
-const BaseCache = require("./base")
-
-const GENERIC = new BaseCache()
-
-exports.CacheKeys = {
- CHECKLIST: "checklist",
- INSTALLATION: "installation",
- ANALYTICS_ENABLED: "analyticsEnabled",
- UNIQUE_TENANT_ID: "uniqueTenantId",
- EVENTS: "events",
- BACKFILL_METADATA: "backfillMetadata",
-}
-
-exports.TTL = {
- ONE_MINUTE: 600,
- ONE_HOUR: 3600,
- ONE_DAY: 86400,
-}
-
-function performExport(funcName) {
- return (...args) => GENERIC[funcName](...args)
-}
-
-exports.keys = performExport("keys")
-exports.get = performExport("get")
-exports.store = performExport("store")
-exports.delete = performExport("delete")
-exports.withCache = performExport("withCache")
-exports.bustCache = performExport("bustCache")
diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts
new file mode 100644
index 0000000000..d8a54e4a3f
--- /dev/null
+++ b/packages/backend-core/src/cache/generic.ts
@@ -0,0 +1,30 @@
+const BaseCache = require("./base")
+
+const GENERIC = new BaseCache()
+
+export enum CacheKey {
+ CHECKLIST = "checklist",
+ INSTALLATION = "installation",
+ ANALYTICS_ENABLED = "analyticsEnabled",
+ UNIQUE_TENANT_ID = "uniqueTenantId",
+ EVENTS = "events",
+ BACKFILL_METADATA = "backfillMetadata",
+ EVENTS_RATE_LIMIT = "eventsRateLimit",
+}
+
+export enum TTL {
+ ONE_MINUTE = 600,
+ ONE_HOUR = 3600,
+ ONE_DAY = 86400,
+}
+
+function performExport(funcName: string) {
+ return (...args: any) => GENERIC[funcName](...args)
+}
+
+export const keys = performExport("keys")
+export const get = performExport("get")
+export const store = performExport("store")
+export const destroy = performExport("delete")
+export const withCache = performExport("withCache")
+export const bustCache = performExport("bustCache")
diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts
new file mode 100644
index 0000000000..58928c271a
--- /dev/null
+++ b/packages/backend-core/src/cache/index.ts
@@ -0,0 +1,5 @@
+export * as generic from "./generic"
+export * as user from "./user"
+export * as app from "./appMetadata"
+export * as writethrough from "./writethrough"
+export * from "./generic"
diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.js b/packages/backend-core/src/cache/tests/writethrough.spec.js
index 68db24b325..716d3f9c23 100644
--- a/packages/backend-core/src/cache/tests/writethrough.spec.js
+++ b/packages/backend-core/src/cache/tests/writethrough.spec.js
@@ -1,6 +1,6 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
const { Writethrough } = require("../writethrough")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
const tk = require("timekeeper")
const START_DATE = Date.now()
@@ -8,8 +8,8 @@ tk.freeze(START_DATE)
const DELAY = 5000
-const db = dangerousGetDB("test")
-const db2 = dangerousGetDB("test2")
+const db = getDB("test")
+const db2 = getDB("test2")
const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
describe("writethrough", () => {
diff --git a/packages/backend-core/src/cache/user.js b/packages/backend-core/src/cache/user.ts
similarity index 68%
rename from packages/backend-core/src/cache/user.js
rename to packages/backend-core/src/cache/user.ts
index 130da1915e..a128465cd6 100644
--- a/packages/backend-core/src/cache/user.js
+++ b/packages/backend-core/src/cache/user.ts
@@ -1,15 +1,16 @@
-const redis = require("../redis/init")
-const { getTenantId, lookupTenantId, doWithGlobalDB } = require("../tenancy")
-const env = require("../environment")
-const accounts = require("../cloud/accounts")
+import * as redis from "../redis/init"
+import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import env from "../environment"
+import * as accounts from "../cloud/accounts"
+import { Database } from "@budibase/types"
const EXPIRY_SECONDS = 3600
/**
* The default populate user function
*/
-const populateFromDB = async (userId, tenantId) => {
- const user = await doWithGlobalDB(tenantId, db => db.get(userId))
+async function populateFromDB(userId: string, tenantId: string) {
+ const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
user.budibaseAccess = true
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(user.email)
@@ -31,7 +32,11 @@ const populateFromDB = async (userId, tenantId) => {
* @param {*} populateUser function to provide the user for re-caching. default to couch db
* @returns
*/
-exports.getUser = async (userId, tenantId = null, populateUser = null) => {
+export async function getUser(
+ userId: string,
+ tenantId?: string,
+ populateUser?: any
+) {
if (!populateUser) {
populateUser = populateFromDB
}
@@ -47,7 +52,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => {
let user = await client.get(userId)
if (!user) {
user = await populateUser(userId, tenantId)
- client.store(userId, user, EXPIRY_SECONDS)
+ await client.store(userId, user, EXPIRY_SECONDS)
}
if (user && !user.tenantId && tenantId) {
// make sure the tenant ID is always correct/set
@@ -56,7 +61,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => {
return user
}
-exports.invalidateUser = async userId => {
+export async function invalidateUser(userId: string) {
const client = await redis.getUserClient()
await client.delete(userId)
}
diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts
index ec6b1604c8..dc889d5b18 100644
--- a/packages/backend-core/src/cache/writethrough.ts
+++ b/packages/backend-core/src/cache/writethrough.ts
@@ -1,6 +1,7 @@
import BaseCache from "./base"
import { getWritethroughClient } from "../redis/init"
import { logWarn } from "../logging"
+import { Database } from "@budibase/types"
const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null
@@ -18,7 +19,7 @@ async function getCache() {
return CACHE
}
-function makeCacheKey(db: PouchDB.Database, key: string) {
+function makeCacheKey(db: Database, key: string) {
return db.name + key
}
@@ -27,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
}
export async function put(
- db: PouchDB.Database,
+ db: Database,
doc: any,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
@@ -63,7 +64,7 @@ export async function put(
return { ok: true, id: output._id, rev: output._rev }
}
-export async function get(db: PouchDB.Database, id: string): Promise {
+export async function get(db: Database, id: string): Promise {
const cache = await getCache()
const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey)
@@ -76,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise {
}
export async function remove(
- db: PouchDB.Database,
+ db: Database,
docOrId: any,
rev?: any
): Promise {
@@ -94,13 +95,10 @@ export async function remove(
}
export class Writethrough {
- db: PouchDB.Database
+ db: Database
writeRateMs: number
- constructor(
- db: PouchDB.Database,
- writeRateMs: number = DEFAULT_WRITE_RATE_MS
- ) {
+ constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
this.db = db
this.writeRateMs = writeRateMs
}
diff --git a/packages/backend-core/src/cloud/accounts.ts b/packages/backend-core/src/cloud/accounts.ts
index cca7469060..90fa7ab824 100644
--- a/packages/backend-core/src/cloud/accounts.ts
+++ b/packages/backend-core/src/cloud/accounts.ts
@@ -1,6 +1,6 @@
import API from "./api"
import env from "../environment"
-import { Headers } from "../constants"
+import { Header } from "../constants"
import { CloudAccount } from "@budibase/types"
const api = new API(env.ACCOUNT_PORTAL_URL)
@@ -14,7 +14,7 @@ export const getAccount = async (
const response = await api.post(`/api/accounts/search`, {
body: payload,
headers: {
- [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+ [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
@@ -35,7 +35,7 @@ export const getAccountByTenantId = async (
const response = await api.post(`/api/accounts/search`, {
body: payload,
headers: {
- [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+ [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
@@ -50,7 +50,7 @@ export const getAccountByTenantId = async (
export const getStatus = async () => {
const response = await api.get(`/api/status`, {
headers: {
- [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+ [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
},
})
const json = await response.json()
diff --git a/packages/backend-core/src/cloud/api.js b/packages/backend-core/src/cloud/api.js
deleted file mode 100644
index d4d4b6c8bb..0000000000
--- a/packages/backend-core/src/cloud/api.js
+++ /dev/null
@@ -1,42 +0,0 @@
-const fetch = require("node-fetch")
-class API {
- constructor(host) {
- this.host = host
- }
-
- apiCall =
- method =>
- async (url = "", options = {}) => {
- if (!options.headers) {
- options.headers = {}
- }
-
- if (!options.headers["Content-Type"]) {
- options.headers = {
- "Content-Type": "application/json",
- Accept: "application/json",
- ...options.headers,
- }
- }
-
- let json = options.headers["Content-Type"] === "application/json"
-
- const requestOptions = {
- method: method,
- body: json ? JSON.stringify(options.body) : options.body,
- headers: options.headers,
- // TODO: See if this is necessary
- credentials: "include",
- }
-
- return await fetch(`${this.host}${url}`, requestOptions)
- }
-
- post = this.apiCall("POST")
- get = this.apiCall("GET")
- patch = this.apiCall("PATCH")
- del = this.apiCall("DELETE")
- put = this.apiCall("PUT")
-}
-
-module.exports = API
diff --git a/packages/backend-core/src/cloud/api.ts b/packages/backend-core/src/cloud/api.ts
new file mode 100644
index 0000000000..287c447271
--- /dev/null
+++ b/packages/backend-core/src/cloud/api.ts
@@ -0,0 +1,55 @@
+import fetch from "node-fetch"
+
+export = class API {
+ host: string
+
+ constructor(host: string) {
+ this.host = host
+ }
+
+ async apiCall(method: string, url: string, options?: any) {
+ if (!options.headers) {
+ options.headers = {}
+ }
+
+ if (!options.headers["Content-Type"]) {
+ options.headers = {
+ "Content-Type": "application/json",
+ Accept: "application/json",
+ ...options.headers,
+ }
+ }
+
+ let json = options.headers["Content-Type"] === "application/json"
+
+ const requestOptions = {
+ method: method,
+ body: json ? JSON.stringify(options.body) : options.body,
+ headers: options.headers,
+ // TODO: See if this is necessary
+ credentials: "include",
+ }
+
+ return await fetch(`${this.host}${url}`, requestOptions)
+ }
+
+ async post(url: string, options?: any) {
+ return this.apiCall("POST", url, options)
+ }
+
+ async get(url: string, options?: any) {
+ return this.apiCall("GET", url, options)
+ }
+
+ async patch(url: string, options?: any) {
+ return this.apiCall("PATCH", url, options)
+ }
+
+ async del(url: string, options?: any) {
+ return this.apiCall("DELETE", url, options)
+ }
+
+ async put(url: string, options?: any) {
+ return this.apiCall("PUT", url, options)
+ }
+}
diff --git a/packages/backend-core/src/clshooked/index.js b/packages/backend-core/src/clshooked/index.js
deleted file mode 100644
index d69ffdd914..0000000000
--- a/packages/backend-core/src/clshooked/index.js
+++ /dev/null
@@ -1,650 +0,0 @@
-const util = require("util")
-const assert = require("assert")
-const wrapEmitter = require("emitter-listener")
-const async_hooks = require("async_hooks")
-
-const CONTEXTS_SYMBOL = "cls@contexts"
-const ERROR_SYMBOL = "error@context"
-
-const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
-
-let currentUid = -1
-
-module.exports = {
- getNamespace: getNamespace,
- createNamespace: createNamespace,
- destroyNamespace: destroyNamespace,
- reset: reset,
- ERROR_SYMBOL: ERROR_SYMBOL,
-}
-
-function Namespace(name) {
- this.name = name
- // changed in 2.7: no default context
- this.active = null
- this._set = []
- this.id = null
- this._contexts = new Map()
- this._indent = 0
- this._hook = null
-}
-
-Namespace.prototype.set = function set(key, value) {
- if (!this.active) {
- throw new Error(
- "No context available. ns.run() or ns.bind() must be called first."
- )
- }
-
- this.active[key] = value
-
- if (DEBUG_CLS_HOOKED) {
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- indentStr +
- "CONTEXT-SET KEY:" +
- key +
- "=" +
- value +
- " in ns:" +
- this.name +
- " currentUid:" +
- currentUid +
- " active:" +
- util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
- )
- }
-
- return value
-}
-
-Namespace.prototype.get = function get(key) {
- if (!this.active) {
- if (DEBUG_CLS_HOOKED) {
- const asyncHooksCurrentId = async_hooks.currentId()
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
- )
- }
- return undefined
- }
- if (DEBUG_CLS_HOOKED) {
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- indentStr +
- "CONTEXT-GETTING KEY:" +
- key +
- "=" +
- this.active[key] +
- " (" +
- this.name +
- ") currentUid:" +
- currentUid +
- " active:" +
- util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
- )
- debug2(
- `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
- this.active[key]
- } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
- this._set.length
- } active:${util.inspect(this.active)}`
- )
- }
- return this.active[key]
-}
-
-Namespace.prototype.createContext = function createContext() {
- // Prototype inherit existing context if created a new child context within existing context.
- let context = Object.create(this.active ? this.active : Object.prototype)
- context._ns_name = this.name
- context.id = currentUid
-
- if (DEBUG_CLS_HOOKED) {
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-CREATED Context: (${
- this.name
- }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
- this._set.length
- } context:${util.inspect(context, {
- showHidden: true,
- depth: 2,
- colors: true,
- })}`
- )
- }
-
- return context
-}
-
-Namespace.prototype.run = function run(fn) {
- let context = this.createContext()
- this.enter(context)
-
- try {
- if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-RUN BEGIN: (${
- this.name
- }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
- this._set.length
- } context:${util.inspect(context)}`
- )
- }
- fn(context)
- return context
- } catch (exception) {
- if (exception) {
- exception[ERROR_SYMBOL] = context
- }
- throw exception
- } finally {
- if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-RUN END: (${
- this.name
- }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
- this._set.length
- } ${util.inspect(context)}`
- )
- }
- this.exit(context)
- }
-}
-
-Namespace.prototype.runAndReturn = function runAndReturn(fn) {
- let value
- this.run(function (context) {
- value = fn(context)
- })
- return value
-}
-
-/**
- * Uses global Promise and assumes Promise is cls friendly or wrapped already.
- * @param {function} fn
- * @returns {*}
- */
-Namespace.prototype.runPromise = function runPromise(fn) {
- let context = this.createContext()
- this.enter(context)
-
- let promise = fn(context)
- if (!promise || !promise.then || !promise.catch) {
- throw new Error("fn must return a promise.")
- }
-
- if (DEBUG_CLS_HOOKED) {
- debug2(
- "CONTEXT-runPromise BEFORE: (" +
- this.name +
- ") currentUid:" +
- currentUid +
- " len:" +
- this._set.length +
- " " +
- util.inspect(context)
- )
- }
-
- return promise
- .then(result => {
- if (DEBUG_CLS_HOOKED) {
- debug2(
- "CONTEXT-runPromise AFTER then: (" +
- this.name +
- ") currentUid:" +
- currentUid +
- " len:" +
- this._set.length +
- " " +
- util.inspect(context)
- )
- }
- this.exit(context)
- return result
- })
- .catch(err => {
- err[ERROR_SYMBOL] = context
- if (DEBUG_CLS_HOOKED) {
- debug2(
- "CONTEXT-runPromise AFTER catch: (" +
- this.name +
- ") currentUid:" +
- currentUid +
- " len:" +
- this._set.length +
- " " +
- util.inspect(context)
- )
- }
- this.exit(context)
- throw err
- })
-}
-
-Namespace.prototype.bind = function bindFactory(fn, context) {
- if (!context) {
- if (!this.active) {
- context = this.createContext()
- } else {
- context = this.active
- }
- }
-
- let self = this
- return function clsBind() {
- self.enter(context)
- try {
- return fn.apply(this, arguments)
- } catch (exception) {
- if (exception) {
- exception[ERROR_SYMBOL] = context
- }
- throw exception
- } finally {
- self.exit(context)
- }
- }
-}
-
-Namespace.prototype.enter = function enter(context) {
- assert.ok(context, "context must be provided for entering")
- if (DEBUG_CLS_HOOKED) {
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-ENTER: (${
- this.name
- }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
- this._set.length
- } ${util.inspect(context)}`
- )
- }
-
- this._set.push(this.active)
- this.active = context
-}
-
-Namespace.prototype.exit = function exit(context) {
- assert.ok(context, "context must be provided for exiting")
- if (DEBUG_CLS_HOOKED) {
- const asyncHooksCurrentId = async_hooks.executionAsyncId()
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
- debug2(
- `${indentStr}CONTEXT-EXIT: (${
- this.name
- }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
- this._set.length
- } ${util.inspect(context)}`
- )
- }
-
- // Fast path for most exits that are at the top of the stack
- if (this.active === context) {
- assert.ok(this._set.length, "can't remove top context")
- this.active = this._set.pop()
- return
- }
-
- // Fast search in the stack using lastIndexOf
- let index = this._set.lastIndexOf(context)
-
- if (index < 0) {
- if (DEBUG_CLS_HOOKED) {
- debug2(
- "??ERROR?? context exiting but not entered - ignoring: " +
- util.inspect(context)
- )
- }
- assert.ok(
- index >= 0,
- "context not currently entered; can't exit. \n" +
- util.inspect(this) +
- "\n" +
- util.inspect(context)
- )
- } else {
- assert.ok(index, "can't remove top context")
- this._set.splice(index, 1)
- }
-}
-
-Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
- assert.ok(
- emitter.on && emitter.addListener && emitter.emit,
- "can only bind real EEs"
- )
-
- let namespace = this
- let thisSymbol = "context@" + this.name
-
- // Capture the context active at the time the emitter is bound.
- function attach(listener) {
- if (!listener) {
- return
- }
- if (!listener[CONTEXTS_SYMBOL]) {
- listener[CONTEXTS_SYMBOL] = Object.create(null)
- }
-
- listener[CONTEXTS_SYMBOL][thisSymbol] = {
- namespace: namespace,
- context: namespace.active,
- }
- }
-
- // At emit time, bind the listener within the correct context.
- function bind(unwrapped) {
- if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
- return unwrapped
- }
-
- let wrapped = unwrapped
- let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
- Object.keys(unwrappedContexts).forEach(function (name) {
- let thunk = unwrappedContexts[name]
- wrapped = thunk.namespace.bind(wrapped, thunk.context)
- })
- return wrapped
- }
-
- wrapEmitter(emitter, attach, bind)
-}
-
-/**
- * If an error comes out of a namespace, it will have a context attached to it.
- * This function knows how to find it.
- *
- * @param {Error} exception Possibly annotated error.
- */
-Namespace.prototype.fromException = function fromException(exception) {
- return exception[ERROR_SYMBOL]
-}
-
-function getNamespace(name) {
- return process.namespaces[name]
-}
-
-function createNamespace(name) {
- assert.ok(name, "namespace must be given a name.")
-
- if (DEBUG_CLS_HOOKED) {
- debug2(`NS-CREATING NAMESPACE (${name})`)
- }
- let namespace = new Namespace(name)
- namespace.id = currentUid
-
- const hook = async_hooks.createHook({
- init(asyncId, type, triggerId, resource) {
- currentUid = async_hooks.executionAsyncId()
-
- //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
- // let initContext = namespace.active;
- // if(!initContext && triggerId) {
- // let parentContext = namespace._contexts.get(triggerId);
- // if (parentContext) {
- // namespace.active = parentContext;
- // namespace._contexts.set(currentUid, parentContext);
- // if (DEBUG_CLS_HOOKED) {
- // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
- // debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
- // }
- // } else if (DEBUG_CLS_HOOKED) {
- // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
- // debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
- // }
- // }else {
- // namespace._contexts.set(currentUid, namespace.active);
- // if (DEBUG_CLS_HOOKED) {
- // const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
- // debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
- // }
- // }
- if (namespace.active) {
- namespace._contexts.set(asyncId, namespace.active)
-
- if (DEBUG_CLS_HOOKED) {
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} resource:${resource}`
- )
- }
- } else if (currentUid === 0) {
- // CurrentId will be 0 when triggered from C++. Promise events
- // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
- const triggerId = async_hooks.triggerAsyncId()
- const triggerIdContext = namespace._contexts.get(triggerId)
- if (triggerIdContext) {
- namespace._contexts.set(asyncId, triggerIdContext)
- if (DEBUG_CLS_HOOKED) {
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} resource:${resource}`
- )
- }
- } else if (DEBUG_CLS_HOOKED) {
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} resource:${resource}`
- )
- }
- }
-
- if (DEBUG_CLS_HOOKED && type === "PROMISE") {
- debug2(util.inspect(resource, { showHidden: true }))
- const parentId = resource.parentId
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} resource:${resource}`
- )
- }
- },
- before(asyncId) {
- currentUid = async_hooks.executionAsyncId()
- let context
-
- /*
- if(currentUid === 0){
- // CurrentId will be 0 when triggered from C++. Promise events
- // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
- //const triggerId = async_hooks.triggerAsyncId();
- context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
- }else{
- context = namespace._contexts.get(currentUid);
- }
- */
-
- //HACK to work with promises until they are fixed in node > 8.1.1
- context =
- namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
- if (context) {
- if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} context:${util.inspect(context)}`
- )
- namespace._indent += 2
- }
-
- namespace.enter(context)
- } else if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} namespace._contexts:${util.inspect(namespace._contexts, {
- showHidden: true,
- depth: 2,
- colors: true,
- })}`
- )
- namespace._indent += 2
- }
- },
- after(asyncId) {
- currentUid = async_hooks.executionAsyncId()
- let context // = namespace._contexts.get(currentUid);
- /*
- if(currentUid === 0){
- // CurrentId will be 0 when triggered from C++. Promise events
- // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
- //const triggerId = async_hooks.triggerAsyncId();
- context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
- }else{
- context = namespace._contexts.get(currentUid);
- }
- */
- //HACK to work with promises until they are fixed in node > 8.1.1
- context =
- namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
- if (context) {
- if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- namespace._indent -= 2
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} context:${util.inspect(context)}`
- )
- }
-
- namespace.exit(context)
- } else if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- namespace._indent -= 2
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} context:${util.inspect(context)}`
- )
- }
- },
- destroy(asyncId) {
- currentUid = async_hooks.executionAsyncId()
- if (DEBUG_CLS_HOOKED) {
- const triggerId = async_hooks.triggerAsyncId()
- const indentStr = " ".repeat(
- namespace._indent < 0 ? 0 : namespace._indent
- )
- debug2(
- `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
- namespace.active,
- { showHidden: true, depth: 2, colors: true }
- )} context:${util.inspect(namespace._contexts.get(currentUid))}`
- )
- }
-
- namespace._contexts.delete(asyncId)
- },
- })
-
- hook.enable()
- namespace._hook = hook
-
- process.namespaces[name] = namespace
- return namespace
-}
-
-function destroyNamespace(name) {
- let namespace = getNamespace(name)
-
- assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
- assert.ok(
- namespace.id,
- "don't assign to process.namespaces directly! " + util.inspect(namespace)
- )
-
- namespace._hook.disable()
- namespace._contexts = null
- process.namespaces[name] = null
-}
-
-function reset() {
- // must unregister async listeners
- if (process.namespaces) {
- Object.keys(process.namespaces).forEach(function (name) {
- destroyNamespace(name)
- })
- }
- process.namespaces = Object.create(null)
-}
-
-process.namespaces = process.namespaces || {}
-
-//const fs = require('fs');
-function debug2(...args) {
- if (DEBUG_CLS_HOOKED) {
- //fs.writeSync(1, `${util.format(...args)}\n`);
- process._rawDebug(`${util.format(...args)}`)
- }
-}
-
-/*function getFunctionName(fn) {
- if (!fn) {
- return fn;
- }
- if (typeof fn === 'function') {
- if (fn.name) {
- return fn.name;
- }
- return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
- } else if (fn.constructor && fn.constructor.name) {
- return fn.constructor.name;
- }
-}*/
diff --git a/packages/backend-core/src/constants.js b/packages/backend-core/src/constants.js
deleted file mode 100644
index 172e66e603..0000000000
--- a/packages/backend-core/src/constants.js
+++ /dev/null
@@ -1,43 +0,0 @@
-exports.UserStatus = {
- ACTIVE: "active",
- INACTIVE: "inactive",
-}
-
-exports.Cookies = {
- CurrentApp: "budibase:currentapp",
- Auth: "budibase:auth",
- Init: "budibase:init",
- DatasourceAuth: "budibase:datasourceauth",
- OIDC_CONFIG: "budibase:oidc:config",
-}
-
-exports.Headers = {
- API_KEY: "x-budibase-api-key",
- LICENSE_KEY: "x-budibase-license-key",
- API_VER: "x-budibase-api-version",
- APP_ID: "x-budibase-app-id",
- TYPE: "x-budibase-type",
- PREVIEW_ROLE: "x-budibase-role",
- TENANT_ID: "x-budibase-tenant-id",
- TOKEN: "x-budibase-token",
- CSRF_TOKEN: "x-csrf-token",
-}
-
-exports.GlobalRoles = {
- OWNER: "owner",
- ADMIN: "admin",
- BUILDER: "builder",
- WORKSPACE_MANAGER: "workspace_manager",
-}
-
-exports.Configs = {
- SETTINGS: "settings",
- ACCOUNT: "account",
- SMTP: "smtp",
- GOOGLE: "google",
- OIDC: "oidc",
- OIDC_LOGOS: "logos_oidc",
-}
-
-exports.MAX_VALID_DATE = new Date(2147483647000)
-exports.DEFAULT_TENANT_ID = "default"
diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/constants/db.ts
similarity index 56%
rename from packages/backend-core/src/db/constants.ts
rename to packages/backend-core/src/constants/db.ts
index 9c6be25424..92392457d6 100644
--- a/packages/backend-core/src/db/constants.ts
+++ b/packages/backend-core/src/constants/db.ts
@@ -4,13 +4,13 @@ export const UNICODE_MAX = "\ufff0"
/**
* Can be used to create a few different forms of querying a view.
*/
-export enum AutomationViewModes {
+export enum AutomationViewMode {
ALL = "all",
AUTOMATION = "automation",
STATUS = "status",
}
-export enum ViewNames {
+export enum ViewName {
USER_BY_APP = "by_app",
USER_BY_EMAIL = "by_email2",
BY_API_KEY = "by_api_key",
@@ -18,16 +18,24 @@ export enum ViewNames {
LINK = "by_link",
ROUTING = "screen_routes",
AUTOMATION_LOGS = "automation_logs",
+ ACCOUNT_BY_EMAIL = "account_by_email",
+ PLATFORM_USERS_LOWERCASE = "platform_users_lowercase",
+ USER_BY_GROUP = "by_group_user",
+ APP_BACKUP_BY_TRIGGER = "by_trigger",
}
export const DeprecatedViews = {
- [ViewNames.USER_BY_EMAIL]: [
+ [ViewName.USER_BY_EMAIL]: [
// removed due to inaccuracy in view doc filter logic
"by_email",
],
}
-export enum DocumentTypes {
+export enum InternalTable {
+ USER_METADATA = "ta_users",
+}
+
+export enum DocumentType {
USER = "us",
GROUP = "gr",
WORKSPACE = "workspace",
@@ -41,6 +49,25 @@ export enum DocumentTypes {
MIGRATIONS = "migrations",
DEV_INFO = "devinfo",
AUTOMATION_LOG = "log_au",
+ ACCOUNT_METADATA = "acc_metadata",
+ PLUGIN = "plg",
+ DATASOURCE = "datasource",
+ DATASOURCE_PLUS = "datasource_plus",
+ APP_BACKUP = "backup",
+ TABLE = "ta",
+ ROW = "ro",
+ AUTOMATION = "au",
+ LINK = "li",
+ WEBHOOK = "wh",
+ INSTANCE = "inst",
+ LAYOUT = "layout",
+ SCREEN = "screen",
+ QUERY = "query",
+ DEPLOYMENTS = "deployments",
+ METADATA = "metadata",
+ MEM_VIEW = "view",
+ USER_FLAG = "flag",
+ AUTOMATION_METADATA = "meta_au",
}
export const StaticDatabases = {
@@ -62,6 +89,7 @@ export const StaticDatabases = {
},
}
-export const APP_PREFIX = exports.DocumentTypes.APP + exports.SEPARATOR
-export const APP_DEV = exports.DocumentTypes.APP_DEV + exports.SEPARATOR
+export const APP_PREFIX = DocumentType.APP + SEPARATOR
+export const APP_DEV = DocumentType.APP_DEV + SEPARATOR
export const APP_DEV_PREFIX = APP_DEV
+export const BUDIBASE_DATASOURCE_TYPE = "budibase"
diff --git a/packages/backend-core/src/constants/index.ts b/packages/backend-core/src/constants/index.ts
new file mode 100644
index 0000000000..62d5e08e63
--- /dev/null
+++ b/packages/backend-core/src/constants/index.ts
@@ -0,0 +1,2 @@
+export * from "./db"
+export * from "./misc"
diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts
new file mode 100644
index 0000000000..61b3cea1f6
--- /dev/null
+++ b/packages/backend-core/src/constants/misc.ts
@@ -0,0 +1,44 @@
+export enum UserStatus {
+ ACTIVE = "active",
+ INACTIVE = "inactive",
+}
+
+export enum Cookie {
+ CurrentApp = "budibase:currentapp",
+ Auth = "budibase:auth",
+ Init = "budibase:init",
+ ACCOUNT_RETURN_URL = "budibase:account:returnurl",
+ DatasourceAuth = "budibase:datasourceauth",
+ OIDC_CONFIG = "budibase:oidc:config",
+}
+
+export enum Header {
+ API_KEY = "x-budibase-api-key",
+ LICENSE_KEY = "x-budibase-license-key",
+ API_VER = "x-budibase-api-version",
+ APP_ID = "x-budibase-app-id",
+ TYPE = "x-budibase-type",
+ PREVIEW_ROLE = "x-budibase-role",
+ TENANT_ID = "x-budibase-tenant-id",
+ TOKEN = "x-budibase-token",
+ CSRF_TOKEN = "x-csrf-token",
+}
+
+export enum GlobalRole {
+ OWNER = "owner",
+ ADMIN = "admin",
+ BUILDER = "builder",
+ WORKSPACE_MANAGER = "workspace_manager",
+}
+
+export enum Config {
+ SETTINGS = "settings",
+ ACCOUNT = "account",
+ SMTP = "smtp",
+ GOOGLE = "google",
+ OIDC = "oidc",
+ OIDC_LOGOS = "logos_oidc",
+}
+
+export const MAX_VALID_DATE = new Date(2147483647000)
+export const DEFAULT_TENANT_ID = "default"
diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts
new file mode 100644
index 0000000000..f0ccdb97a8
--- /dev/null
+++ b/packages/backend-core/src/context/Context.ts
@@ -0,0 +1,17 @@
+import { AsyncLocalStorage } from "async_hooks"
+
+export default class Context {
+ static storage = new AsyncLocalStorage>()
+
+ static run(context: Record, func: any) {
+ return Context.storage.run(context, () => func())
+ }
+
+ static get(): Record {
+ return Context.storage.getStore() as Record
+ }
+
+ static set(context: Record) {
+ Context.storage.enterWith(context)
+ }
+}
diff --git a/packages/backend-core/src/context/FunctionContext.js b/packages/backend-core/src/context/FunctionContext.js
deleted file mode 100644
index c0ed34fe78..0000000000
--- a/packages/backend-core/src/context/FunctionContext.js
+++ /dev/null
@@ -1,47 +0,0 @@
-const cls = require("../clshooked")
-const { newid } = require("../hashing")
-
-const REQUEST_ID_KEY = "requestId"
-const MAIN_CTX = cls.createNamespace("main")
-
-function getContextStorage(namespace) {
- if (namespace && namespace.active) {
- let contextData = namespace.active
- delete contextData.id
- delete contextData._ns_name
- return contextData
- }
- return {}
-}
-
-class FunctionContext {
- static run(callback) {
- return MAIN_CTX.runAndReturn(async () => {
- const namespaceId = newid()
- MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
- const namespace = cls.createNamespace(namespaceId)
- let response = await namespace.runAndReturn(callback)
- cls.destroyNamespace(namespaceId)
- return response
- })
- }
-
- static setOnContext(key, value) {
- const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
- const namespace = cls.getNamespace(namespaceId)
- namespace.set(key, value)
- }
-
- static getFromContext(key) {
- const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
- const namespace = cls.getNamespace(namespaceId)
- const context = getContextStorage(namespace)
- if (context) {
- return context[key]
- } else {
- return null
- }
- }
-}
-
-module.exports = FunctionContext
diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts
deleted file mode 100644
index ef8dcd7821..0000000000
--- a/packages/backend-core/src/context/constants.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-export enum ContextKeys {
- TENANT_ID = "tenantId",
- GLOBAL_DB = "globalDb",
- APP_ID = "appId",
- IDENTITY = "identity",
- // whatever the request app DB was
- CURRENT_DB = "currentDb",
- // get the prod app DB from the request
- PROD_DB = "prodDb",
- // get the dev app DB from the request
- DEV_DB = "devDb",
- DB_OPTS = "dbOpts",
- // check if something else is using the context, don't close DB
- TENANCY_IN_USE = "tenancyInUse",
- APP_IN_USE = "appInUse",
- IDENTITY_IN_USE = "identityInUse",
-}
diff --git a/packages/backend-core/src/context/deprovision.js b/packages/backend-core/src/context/deprovision.ts
similarity index 58%
rename from packages/backend-core/src/context/deprovision.js
rename to packages/backend-core/src/context/deprovision.ts
index ba3c2d8449..81f03096dc 100644
--- a/packages/backend-core/src/context/deprovision.js
+++ b/packages/backend-core/src/context/deprovision.ts
@@ -1,15 +1,19 @@
-const { getGlobalUserParams, getAllApps } = require("../db/utils")
-const { doWithDB } = require("../db")
-const { doWithGlobalDB } = require("../tenancy")
-const { StaticDatabases } = require("../db/constants")
+import {
+ getGlobalUserParams,
+ getAllApps,
+ doWithDB,
+ StaticDatabases,
+} from "../db"
+import { doWithGlobalDB } from "../tenancy"
+import { App, Tenants, User, Database } from "@budibase/types"
const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
-const removeTenantFromInfoDB = async tenantId => {
+async function removeTenantFromInfoDB(tenantId: string) {
try {
- await doWithDB(PLATFORM_INFO_DB, async infoDb => {
- let tenants = await infoDb.get(TENANT_DOC)
+ await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
+ const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
await infoDb.put(tenants)
@@ -20,14 +24,14 @@ const removeTenantFromInfoDB = async tenantId => {
}
}
-exports.removeUserFromInfoDB = async dbUser => {
- await doWithDB(PLATFORM_INFO_DB, async infoDb => {
- const keys = [dbUser._id, dbUser.email]
+export async function removeUserFromInfoDB(dbUser: User) {
+ await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
+ const keys = [dbUser._id!, dbUser.email]
const userDocs = await infoDb.allDocs({
keys,
include_docs: true,
})
- const toDelete = userDocs.rows.map(row => {
+ const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
@@ -37,18 +41,18 @@ exports.removeUserFromInfoDB = async dbUser => {
})
}
-const removeUsersFromInfoDB = async tenantId => {
- return doWithGlobalDB(tenantId, async db => {
+async function removeUsersFromInfoDB(tenantId: string) {
+ return doWithGlobalDB(tenantId, async (db: any) => {
try {
const allUsers = await db.allDocs(
getGlobalUserParams(null, {
include_docs: true,
})
)
- await doWithDB(PLATFORM_INFO_DB, async infoDb => {
- const allEmails = allUsers.rows.map(row => row.doc.email)
+ await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
+ const allEmails = allUsers.rows.map((row: any) => row.doc.email)
// get the id docs
- let keys = allUsers.rows.map(row => row.id)
+ let keys = allUsers.rows.map((row: any) => row.id)
// and the email docs
keys = keys.concat(allEmails)
// retrieve the docs and delete them
@@ -56,7 +60,7 @@ const removeUsersFromInfoDB = async tenantId => {
keys,
include_docs: true,
})
- const toDelete = userDocs.rows.map(row => {
+ const toDelete = userDocs.rows.map((row: any) => {
return {
...row.doc,
_deleted: true,
@@ -71,8 +75,8 @@ const removeUsersFromInfoDB = async tenantId => {
})
}
-const removeGlobalDB = async tenantId => {
- return doWithGlobalDB(tenantId, async db => {
+async function removeGlobalDB(tenantId: string) {
+ return doWithGlobalDB(tenantId, async (db: Database) => {
try {
await db.destroy()
} catch (err) {
@@ -82,11 +86,11 @@ const removeGlobalDB = async tenantId => {
})
}
-const removeTenantApps = async tenantId => {
+async function removeTenantApps(tenantId: string) {
try {
- const apps = await getAllApps({ all: true })
+ const apps = (await getAllApps({ all: true })) as App[]
const destroyPromises = apps.map(app =>
- doWithDB(app.appId, db => db.destroy())
+ doWithDB(app.appId, (db: Database) => db.destroy())
)
await Promise.allSettled(destroyPromises)
} catch (err) {
@@ -96,7 +100,7 @@ const removeTenantApps = async tenantId => {
}
// can't live in tenancy package due to circular dependency on db/utils
-exports.deleteTenant = async tenantId => {
+export async function deleteTenant(tenantId: string) {
await removeTenantFromInfoDB(tenantId)
await removeUsersFromInfoDB(tenantId)
await removeGlobalDB(tenantId)
diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts
index 37e1ecf40a..648dd1b5fd 100644
--- a/packages/backend-core/src/context/identity.ts
+++ b/packages/backend-core/src/context/identity.ts
@@ -2,23 +2,22 @@ import {
IdentityContext,
IdentityType,
User,
- UserContext,
isCloudAccount,
Account,
AccountUserContext,
} from "@budibase/types"
import * as context from "."
-export const getIdentity = (): IdentityContext | undefined => {
+export function getIdentity(): IdentityContext | undefined {
return context.getIdentity()
}
-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+export function doInIdentityContext(identity: IdentityContext, task: any) {
return context.doInIdentityContext(identity, task)
}
-export const doInUserContext = (user: User, task: any) => {
- const userContext: UserContext = {
+export function doInUserContext(user: User, task: any) {
+ const userContext: any = {
...user,
_id: user._id as string,
type: IdentityType.USER,
@@ -26,7 +25,7 @@ export const doInUserContext = (user: User, task: any) => {
return doInIdentityContext(userContext, task)
}
-export const doInAccountContext = (account: Account, task: any) => {
+export function doInAccountContext(account: Account, task: any) {
const _id = getAccountUserId(account)
const tenantId = account.tenantId
const accountContext: AccountUserContext = {
@@ -38,12 +37,12 @@ export const doInAccountContext = (account: Account, task: any) => {
return doInIdentityContext(accountContext, task)
}
-export const getAccountUserId = (account: Account) => {
+export function getAccountUserId(account: Account) {
let userId: string
if (isCloudAccount(account)) {
userId = account.budibaseUserId
} else {
- // use account id as user id for self hosting
+ // use account id as user id for self-hosting
userId = account.accountId
}
return userId
diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts
index 1e430f01de..9c70363170 100644
--- a/packages/backend-core/src/context/index.ts
+++ b/packages/backend-core/src/context/index.ts
@@ -1,251 +1,3 @@
-import env from "../environment"
-import { SEPARATOR, DocumentTypes } from "../db/constants"
-import cls from "./FunctionContext"
-import { dangerousGetDB, closeDB } from "../db"
-import { baseGlobalDBName } from "../tenancy/utils"
-import { IdentityContext } from "@budibase/types"
-import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
-import { ContextKeys } from "./constants"
-import {
- updateUsing,
- closeWithUsing,
- setAppTenantId,
- setIdentity,
- closeAppDBs,
- getContextDB,
-} from "./utils"
-
-export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
-
-// some test cases call functions directly, need to
-// store an app ID to pretend there is a context
-let TEST_APP_ID: string | null = null
-
-export const closeTenancy = async () => {
- let db
- try {
- if (env.USE_COUCH) {
- db = getGlobalDB()
- }
- } catch (err) {
- // no DB found - skip closing
- return
- }
- await closeDB(db)
- // clear from context now that database is closed/task is finished
- cls.setOnContext(ContextKeys.TENANT_ID, null)
- cls.setOnContext(ContextKeys.GLOBAL_DB, null)
-}
-
-// export const isDefaultTenant = () => {
-// return getTenantId() === DEFAULT_TENANT_ID
-// }
-
-export const isMultiTenant = () => {
- return env.MULTI_TENANCY
-}
-
-/**
- * Given an app ID this will attempt to retrieve the tenant ID from it.
- * @return {null|string} The tenant ID found within the app ID.
- */
-export const getTenantIDFromAppID = (appId: string) => {
- if (!appId) {
- return null
- }
- const split = appId.split(SEPARATOR)
- const hasDev = split[1] === DocumentTypes.DEV
- if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
- return null
- }
- if (hasDev) {
- return split[2]
- } else {
- return split[1]
- }
-}
-
-// used for automations, API endpoints should always be in context already
-export const doInTenant = (tenantId: string | null, task: any) => {
- // make sure default always selected in single tenancy
- if (!env.MULTI_TENANCY) {
- tenantId = tenantId || DEFAULT_TENANT_ID
- }
- // the internal function is so that we can re-use an existing
- // context - don't want to close DB on a parent context
- async function internal(opts = { existing: false }) {
- // set the tenant id + global db if this is a new context
- if (!opts.existing) {
- updateTenantId(tenantId)
- }
-
- try {
- // invoke the task
- return await task()
- } finally {
- await closeWithUsing(ContextKeys.TENANCY_IN_USE, () => {
- return closeTenancy()
- })
- }
- }
-
- const existing = cls.getFromContext(ContextKeys.TENANT_ID) === tenantId
- return updateUsing(ContextKeys.TENANCY_IN_USE, existing, internal)
-}
-
-export const doInAppContext = (appId: string, task: any) => {
- if (!appId) {
- throw new Error("appId is required")
- }
-
- const identity = getIdentity()
-
- // the internal function is so that we can re-use an existing
- // context - don't want to close DB on a parent context
- async function internal(opts = { existing: false }) {
- // set the app tenant id
- if (!opts.existing) {
- setAppTenantId(appId)
- }
- // set the app ID
- cls.setOnContext(ContextKeys.APP_ID, appId)
-
- // preserve the identity
- if (identity) {
- setIdentity(identity)
- }
- try {
- // invoke the task
- return await task()
- } finally {
- await closeWithUsing(ContextKeys.APP_IN_USE, async () => {
- await closeAppDBs()
- await closeTenancy()
- })
- }
- }
- const existing = cls.getFromContext(ContextKeys.APP_ID) === appId
- return updateUsing(ContextKeys.APP_IN_USE, existing, internal)
-}
-
-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
- if (!identity) {
- throw new Error("identity is required")
- }
-
- async function internal(opts = { existing: false }) {
- if (!opts.existing) {
- cls.setOnContext(ContextKeys.IDENTITY, identity)
- // set the tenant so that doInTenant will preserve identity
- if (identity.tenantId) {
- updateTenantId(identity.tenantId)
- }
- }
-
- try {
- // invoke the task
- return await task()
- } finally {
- await closeWithUsing(ContextKeys.IDENTITY_IN_USE, async () => {
- setIdentity(null)
- await closeTenancy()
- })
- }
- }
-
- const existing = cls.getFromContext(ContextKeys.IDENTITY)
- return updateUsing(ContextKeys.IDENTITY_IN_USE, existing, internal)
-}
-
-export const getIdentity = (): IdentityContext | undefined => {
- try {
- return cls.getFromContext(ContextKeys.IDENTITY)
- } catch (e) {
- // do nothing - identity is not in context
- }
-}
-
-export const updateTenantId = (tenantId: string | null) => {
- cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
- if (env.USE_COUCH) {
- setGlobalDB(tenantId)
- }
-}
-
-export const updateAppId = async (appId: string) => {
- try {
- // have to close first, before removing the databases from context
- await closeAppDBs()
- cls.setOnContext(ContextKeys.APP_ID, appId)
- } catch (err) {
- if (env.isTest()) {
- TEST_APP_ID = appId
- } else {
- throw err
- }
- }
-}
-
-export const setGlobalDB = (tenantId: string | null) => {
- const dbName = baseGlobalDBName(tenantId)
- const db = dangerousGetDB(dbName)
- cls.setOnContext(ContextKeys.GLOBAL_DB, db)
- return db
-}
-
-export const getGlobalDB = () => {
- const db = cls.getFromContext(ContextKeys.GLOBAL_DB)
- if (!db) {
- throw new Error("Global DB not found")
- }
- return db
-}
-
-export const isTenantIdSet = () => {
- const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
- return !!tenantId
-}
-
-export const getTenantId = () => {
- if (!isMultiTenant()) {
- return DEFAULT_TENANT_ID
- }
- const tenantId = cls.getFromContext(ContextKeys.TENANT_ID)
- if (!tenantId) {
- throw new Error("Tenant id not found")
- }
- return tenantId
-}
-
-export const getAppId = () => {
- const foundId = cls.getFromContext(ContextKeys.APP_ID)
- if (!foundId && env.isTest() && TEST_APP_ID) {
- return TEST_APP_ID
- } else {
- return foundId
- }
-}
-
-/**
- * Opens the app database based on whatever the request
- * contained, dev or prod.
- */
-export const getAppDB = (opts?: any) => {
- return getContextDB(ContextKeys.CURRENT_DB, opts)
-}
-
-/**
- * This specifically gets the prod app ID, if the request
- * contained a development app ID, this will open the prod one.
- */
-export const getProdAppDB = (opts?: any) => {
- return getContextDB(ContextKeys.PROD_DB, opts)
-}
-
-/**
- * This specifically gets the dev app ID, if the request
- * contained a prod app ID, this will open the dev one.
- */
-export const getDevAppDB = (opts?: any) => {
- return getContextDB(ContextKeys.DEV_DB, opts)
-}
+export { DEFAULT_TENANT_ID } from "../constants"
+export * as identity from "./identity"
+export * from "./mainContext"
diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts
new file mode 100644
index 0000000000..d743d2f49b
--- /dev/null
+++ b/packages/backend-core/src/context/mainContext.ts
@@ -0,0 +1,245 @@
+// some test cases call functions directly, need to
+// store an app ID to pretend there is a context
+import env from "../environment"
+import Context from "./Context"
+import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
+import { getDB } from "../db/db"
+import {
+ DocumentType,
+ SEPARATOR,
+ StaticDatabases,
+ DEFAULT_TENANT_ID,
+} from "../constants"
+import { Database, IdentityContext } from "@budibase/types"
+
+export type ContextMap = {
+ tenantId?: string
+ appId?: string
+ identity?: IdentityContext
+}
+
+let TEST_APP_ID: string | null = null
+
+export function getGlobalDBName(tenantId?: string) {
+ // tenant ID can be set externally, for example user API where
+ // new tenants are being created, this may be the case
+ if (!tenantId) {
+ tenantId = getTenantId()
+ }
+ return baseGlobalDBName(tenantId)
+}
+
+export function baseGlobalDBName(tenantId: string | undefined | null) {
+ let dbName
+ if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+ dbName = StaticDatabases.GLOBAL.name
+ } else {
+ dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+ }
+ return dbName
+}
+
+export function isMultiTenant() {
+ return env.MULTI_TENANCY
+}
+
+export function isTenantIdSet() {
+ const context = Context.get()
+ return !!context?.tenantId
+}
+
+export function isTenancyEnabled() {
+ return env.MULTI_TENANCY
+}
+
+/**
+ * Given an app ID this will attempt to retrieve the tenant ID from it.
+ * @return {null|string} The tenant ID found within the app ID.
+ */
+export function getTenantIDFromAppID(appId: string) {
+ if (!appId) {
+ return undefined
+ }
+ if (!isMultiTenant()) {
+ return DEFAULT_TENANT_ID
+ }
+ const split = appId.split(SEPARATOR)
+ const hasDev = split[1] === DocumentType.DEV
+ if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
+ return undefined
+ }
+ if (hasDev) {
+ return split[2]
+ } else {
+ return split[1]
+ }
+}
+
+function updateContext(updates: ContextMap) {
+ let context: ContextMap
+ try {
+ context = Context.get()
+ } catch (err) {
+ // no context, start empty
+ context = {}
+ }
+ context = {
+ ...context,
+ ...updates,
+ }
+ return context
+}
+
+async function newContext(updates: ContextMap, task: any) {
+ // see if there already is a context setup
+ let context: ContextMap = updateContext(updates)
+ return Context.run(context, task)
+}
+
+export async function doInContext(appId: string, task: any): Promise {
+ const tenantId = getTenantIDFromAppID(appId)
+ return newContext(
+ {
+ tenantId,
+ appId,
+ },
+ task
+ )
+}
+
+export async function doInTenant(
+ tenantId: string | null,
+ task: any
+): Promise {
+ // make sure default always selected in single tenancy
+ if (!env.MULTI_TENANCY) {
+ tenantId = tenantId || DEFAULT_TENANT_ID
+ }
+
+ const updates = tenantId ? { tenantId } : {}
+ return newContext(updates, task)
+}
+
+export async function doInAppContext(appId: string, task: any): Promise {
+ if (!appId) {
+ throw new Error("appId is required")
+ }
+
+ const tenantId = getTenantIDFromAppID(appId)
+ const updates: ContextMap = { appId }
+ if (tenantId) {
+ updates.tenantId = tenantId
+ }
+ return newContext(updates, task)
+}
+
+export async function doInIdentityContext(
+ identity: IdentityContext,
+ task: any
+): Promise {
+ if (!identity) {
+ throw new Error("identity is required")
+ }
+
+ const context: ContextMap = {
+ identity,
+ }
+ if (identity.tenantId) {
+ context.tenantId = identity.tenantId
+ }
+ return newContext(context, task)
+}
+
+export function getIdentity(): IdentityContext | undefined {
+ try {
+ const context = Context.get()
+ return context?.identity
+ } catch (e) {
+ // do nothing - identity is not in context
+ }
+}
+
+export function getTenantId(): string {
+ if (!isMultiTenant()) {
+ return DEFAULT_TENANT_ID
+ }
+ const context = Context.get()
+ const tenantId = context?.tenantId
+ if (!tenantId) {
+ throw new Error("Tenant id not found")
+ }
+ return tenantId
+}
+
+export function getAppId(): string | undefined {
+ const context = Context.get()
+ const foundId = context?.appId
+ if (!foundId && env.isTest() && TEST_APP_ID) {
+ return TEST_APP_ID
+ } else {
+ return foundId
+ }
+}
+
+export function updateTenantId(tenantId?: string) {
+ let context: ContextMap = updateContext({
+ tenantId,
+ })
+ Context.set(context)
+}
+
+export function updateAppId(appId: string) {
+ let context: ContextMap = updateContext({
+ appId,
+ })
+ try {
+ Context.set(context)
+ } catch (err) {
+ if (env.isTest()) {
+ TEST_APP_ID = appId
+ } else {
+ throw err
+ }
+ }
+}
+
+export function getGlobalDB(): Database {
+ const context = Context.get()
+ if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
+ throw new Error("Global DB not found")
+ }
+ return getDB(baseGlobalDBName(context?.tenantId))
+}
+
+/**
+ * Gets the app database based on whatever the request
+ * contained, dev or prod.
+ */
+export function getAppDB(opts?: any): Database {
+ const appId = getAppId()
+ return getDB(appId, opts)
+}
+
+/**
+ * This specifically gets the prod app ID, if the request
+ * contained a development app ID, this will get the prod one.
+ */
+export function getProdAppDB(opts?: any): Database {
+ const appId = getAppId()
+ if (!appId) {
+ throw new Error("Unable to retrieve prod DB - no app ID.")
+ }
+ return getDB(getProdAppID(appId), opts)
+}
+
+/**
+ * This specifically gets the dev app ID, if the request
+ * contained a prod app ID, this will get the dev one.
+ */
+export function getDevAppDB(opts?: any): Database {
+ const appId = getAppId()
+ if (!appId) {
+ throw new Error("Unable to retrieve dev DB - no app ID.")
+ }
+ return getDB(getDevelopmentAppID(appId), opts)
+}
diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.js
similarity index 80%
rename from packages/backend-core/src/context/tests/index.spec.ts
rename to packages/backend-core/src/context/tests/index.spec.js
index 55ecd333a3..ea60806d21 100644
--- a/packages/backend-core/src/context/tests/index.spec.ts
+++ b/packages/backend-core/src/context/tests/index.spec.js
@@ -1,18 +1,9 @@
-import "../../../tests/utilities/TestConfiguration"
-import * as context from ".."
-import { DEFAULT_TENANT_ID } from "../../constants"
-import env from "../../environment"
-
-// must use require to spy index file exports due to known issue in jest
-const dbUtils = require("../../db")
-jest.spyOn(dbUtils, "closeDB")
-jest.spyOn(dbUtils, "dangerousGetDB")
+require("../../../tests")
+const context = require("../")
+const { DEFAULT_TENANT_ID } = require("../../constants")
+const env = require("../../environment")
describe("context", () => {
- beforeEach(() => {
- jest.clearAllMocks()
- })
-
describe("doInTenant", () => {
describe("single-tenancy", () => {
it("defaults to the default tenant", () => {
@@ -25,8 +16,6 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("global-db")
})
- expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
- expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
})
@@ -40,7 +29,7 @@ describe("context", () => {
let error
try {
context.getTenantId()
- } catch (e: any) {
+ } catch (e) {
error = e
}
expect(error.message).toBe("Tenant id not found")
@@ -59,7 +48,7 @@ describe("context", () => {
let error
try {
context.getGlobalDB()
- } catch (e: any) {
+ } catch (e) {
error = e
}
expect(error.message).toBe("Global DB not found")
@@ -85,8 +74,6 @@ describe("context", () => {
const db = context.getGlobalDB()
expect(db.name).toBe("test_global-db")
})
- expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
- expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets the tenant id when nested with same tenant id", async () => {
@@ -121,10 +108,6 @@ describe("context", () => {
})
})
})
-
- // only 1 db is opened and closed
- expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
- expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
})
it("sets different tenant id inside another context", () => {
diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts
deleted file mode 100644
index 62693f18e8..0000000000
--- a/packages/backend-core/src/context/utils.ts
+++ /dev/null
@@ -1,113 +0,0 @@
-import {
- DEFAULT_TENANT_ID,
- getAppId,
- getTenantIDFromAppID,
- updateTenantId,
-} from "./index"
-import cls from "./FunctionContext"
-import { IdentityContext } from "@budibase/types"
-import { ContextKeys } from "./constants"
-import { dangerousGetDB, closeDB } from "../db"
-import { isEqual } from "lodash"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
-import env from "../environment"
-
-export async function updateUsing(
- usingKey: string,
- existing: boolean,
- internal: (opts: { existing: boolean }) => Promise
-) {
- const using = cls.getFromContext(usingKey)
- if (using && existing) {
- cls.setOnContext(usingKey, using + 1)
- return internal({ existing: true })
- } else {
- return cls.run(async () => {
- cls.setOnContext(usingKey, 1)
- return internal({ existing: false })
- })
- }
-}
-
-export async function closeWithUsing(
- usingKey: string,
- closeFn: () => Promise
-) {
- const using = cls.getFromContext(usingKey)
- if (!using || using <= 1) {
- await closeFn()
- } else {
- cls.setOnContext(usingKey, using - 1)
- }
-}
-
-export const setAppTenantId = (appId: string) => {
- const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
- updateTenantId(appTenantId)
-}
-
-export const setIdentity = (identity: IdentityContext | null) => {
- cls.setOnContext(ContextKeys.IDENTITY, identity)
-}
-
-// this function makes sure the PouchDB objects are closed and
-// fully deleted when finished - this protects against memory leaks
-export async function closeAppDBs() {
- const dbKeys = [
- ContextKeys.CURRENT_DB,
- ContextKeys.PROD_DB,
- ContextKeys.DEV_DB,
- ]
- for (let dbKey of dbKeys) {
- const db = cls.getFromContext(dbKey)
- if (!db) {
- continue
- }
- await closeDB(db)
- // clear the DB from context, incase someone tries to use it again
- cls.setOnContext(dbKey, null)
- }
- // clear the app ID now that the databases are closed
- if (cls.getFromContext(ContextKeys.APP_ID)) {
- cls.setOnContext(ContextKeys.APP_ID, null)
- }
- if (cls.getFromContext(ContextKeys.DB_OPTS)) {
- cls.setOnContext(ContextKeys.DB_OPTS, null)
- }
-}
-
-export function getContextDB(key: string, opts: any) {
- const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
- let storedOpts = cls.getFromContext(dbOptsKey)
- let db = cls.getFromContext(key)
- if (db && isEqual(opts, storedOpts)) {
- return db
- }
-
- const appId = getAppId()
- let toUseAppId
-
- switch (key) {
- case ContextKeys.CURRENT_DB:
- toUseAppId = appId
- break
- case ContextKeys.PROD_DB:
- toUseAppId = getProdAppID(appId)
- break
- case ContextKeys.DEV_DB:
- toUseAppId = getDevelopmentAppID(appId)
- break
- }
- db = dangerousGetDB(toUseAppId, opts)
- try {
- cls.setOnContext(key, db)
- if (opts) {
- cls.setOnContext(dbOptsKey, opts)
- }
- } catch (err) {
- if (!env.isTest()) {
- throw err
- }
- }
- return db
-}
diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts
index b46f6072be..eb9d613a58 100644
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@@ -1,4 +1,5 @@
-import { dangerousGetDB, closeDB } from "."
+import { getPouchDB, closePouchDB } from "./couch"
+import { DocumentType } from "../constants"
class Replication {
source: any
@@ -11,12 +12,12 @@ class Replication {
* @param {String} target - the DB you want to replicate to, or rollback from
*/
constructor({ source, target }: any) {
- this.source = dangerousGetDB(source)
- this.target = dangerousGetDB(target)
+ this.source = getPouchDB(source)
+ this.target = getPouchDB(target)
}
close() {
- return Promise.all([closeDB(this.source), closeDB(this.target)])
+ return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
}
promisify(operation: any, opts = {}) {
@@ -53,13 +54,22 @@ class Replication {
return this.replication
}
+ appReplicateOpts() {
+ return {
+ filter: (doc: any) => {
+ return doc._id !== DocumentType.APP_METADATA
+ },
+ }
+ }
+
/**
* Rollback the target DB back to the state of the source DB
*/
async rollback() {
await this.target.destroy()
// Recreate the DB again
- this.target = dangerousGetDB(this.target.name)
+ this.target = getPouchDB(this.target.name)
+ // take the opportunity to remove deleted tombstones
await this.replicate()
}
diff --git a/packages/backend-core/src/db/conversions.js b/packages/backend-core/src/db/conversions.ts
similarity index 68%
rename from packages/backend-core/src/db/conversions.js
rename to packages/backend-core/src/db/conversions.ts
index 90c04e9251..381c5cb90f 100644
--- a/packages/backend-core/src/db/conversions.js
+++ b/packages/backend-core/src/db/conversions.ts
@@ -1,32 +1,33 @@
+import { APP_DEV_PREFIX, APP_PREFIX } from "../constants"
+import { App } from "@budibase/types"
const NO_APP_ERROR = "No app provided"
-const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants")
-exports.isDevAppID = appId => {
+export function isDevAppID(appId?: string) {
if (!appId) {
throw NO_APP_ERROR
}
return appId.startsWith(APP_DEV_PREFIX)
}
-exports.isProdAppID = appId => {
+export function isProdAppID(appId?: string) {
if (!appId) {
throw NO_APP_ERROR
}
- return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId)
+ return appId.startsWith(APP_PREFIX) && !isDevAppID(appId)
}
-exports.isDevApp = app => {
+export function isDevApp(app: App) {
if (!app) {
throw NO_APP_ERROR
}
- return exports.isDevAppID(app.appId)
+ return isDevAppID(app.appId)
}
/**
* Generates a development app ID from a real app ID.
* @returns {string} the dev app ID which can be used for dev database.
*/
-exports.getDevelopmentAppID = appId => {
+export function getDevelopmentAppID(appId: string) {
if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
@@ -36,11 +37,12 @@ exports.getDevelopmentAppID = appId => {
const rest = split.join(APP_PREFIX)
return `${APP_DEV_PREFIX}${rest}`
}
+export const getDevAppID = getDevelopmentAppID
/**
* Convert a development app ID to a deployed app ID.
*/
-exports.getProdAppID = appId => {
+export function getProdAppID(appId: string) {
if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
return appId
}
@@ -51,7 +53,7 @@ exports.getProdAppID = appId => {
return `${APP_PREFIX}${rest}`
}
-exports.extractAppUUID = id => {
+export function extractAppUUID(id: string) {
const split = id?.split("_") || []
return split.length ? split[split.length - 1] : null
}
diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
new file mode 100644
index 0000000000..de06b4e8ee
--- /dev/null
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -0,0 +1,193 @@
+import Nano from "nano"
+import {
+ AllDocsResponse,
+ AnyDocument,
+ Database,
+ DatabaseOpts,
+ DatabaseQueryOpts,
+ DatabasePutOpts,
+ DatabaseCreateIndexOpts,
+ DatabaseDeleteIndexOpts,
+ Document,
+ isDocument,
+} from "@budibase/types"
+import { getCouchInfo } from "./connections"
+import { directCouchCall } from "./utils"
+import { getPouchDB } from "./pouchDB"
+import { WriteStream, ReadStream } from "fs"
+
+export class DatabaseImpl implements Database {
+ public readonly name: string
+ private static nano: Nano.ServerScope
+ private readonly pouchOpts: DatabaseOpts
+
+ constructor(dbName?: string, opts?: DatabaseOpts) {
+ if (dbName == null) {
+ throw new Error("Database name cannot be undefined.")
+ }
+ this.name = dbName
+ this.pouchOpts = opts || {}
+ if (!DatabaseImpl.nano) {
+ DatabaseImpl.init()
+ }
+ }
+
+ static init() {
+ const couchInfo = getCouchInfo()
+ DatabaseImpl.nano = Nano({
+ url: couchInfo.url,
+ requestDefaults: {
+ headers: {
+ Authorization: couchInfo.cookie,
+ },
+ },
+ parseUrl: false,
+ })
+ }
+
+ async exists() {
+ let response = await directCouchCall(`/${this.name}`, "HEAD")
+ return response.status === 200
+ }
+
+ async checkSetup() {
+ let shouldCreate = !this.pouchOpts?.skip_setup
+ // check exists in a lightweight fashion
+ let exists = await this.exists()
+ if (!shouldCreate && !exists) {
+ throw new Error("DB does not exist")
+ }
+ if (!exists) {
+ await DatabaseImpl.nano.db.create(this.name)
+ }
+ return DatabaseImpl.nano.db.use(this.name)
+ }
+
+ private async updateOutput(fnc: any) {
+ try {
+ return await fnc()
+ } catch (err: any) {
+ if (err.statusCode) {
+ err.status = err.statusCode
+ }
+ throw err
+ }
+ }
+
+ async get(id?: string): Promise {
+ const db = await this.checkSetup()
+ if (!id) {
+ throw new Error("Unable to get doc without a valid _id.")
+ }
+ return this.updateOutput(() => db.get(id))
+ }
+
+ async remove(idOrDoc: string | Document, rev?: string) {
+ const db = await this.checkSetup()
+ let _id: string
+ let _rev: string
+
+ if (isDocument(idOrDoc)) {
+ _id = idOrDoc._id!
+ _rev = idOrDoc._rev!
+ } else {
+ _id = idOrDoc
+ _rev = rev!
+ }
+
+ if (!_id || !_rev) {
+ throw new Error("Unable to remove doc without a valid _id and _rev.")
+ }
+ return this.updateOutput(() => db.destroy(_id, _rev))
+ }
+
+ async put(document: AnyDocument, opts?: DatabasePutOpts) {
+ if (!document._id) {
+ throw new Error("Cannot store document without _id field.")
+ }
+ const db = await this.checkSetup()
+ if (!document.createdAt) {
+ document.createdAt = new Date().toISOString()
+ }
+ document.updatedAt = new Date().toISOString()
+ if (opts?.force && document._id) {
+ try {
+ const existing = await this.get(document._id)
+ if (existing) {
+ document._rev = existing._rev
+ }
+ } catch (err: any) {
+ if (err.status !== 404) {
+ throw err
+ }
+ }
+ }
+ return this.updateOutput(() => db.insert(document))
+ }
+
+ async bulkDocs(documents: AnyDocument[]) {
+ const db = await this.checkSetup()
+ return this.updateOutput(() => db.bulk({ docs: documents }))
+ }
+
+ async allDocs(params: DatabaseQueryOpts): Promise> {
+ const db = await this.checkSetup()
+ return this.updateOutput(() => db.list(params))
+ }
+
+ async query(
+ viewName: string,
+ params: DatabaseQueryOpts
+ ): Promise> {
+ const db = await this.checkSetup()
+ const [database, view] = viewName.split("/")
+ return this.updateOutput(() => db.view(database, view, params))
+ }
+
+ async destroy() {
+ try {
+ await DatabaseImpl.nano.db.destroy(this.name)
+ } catch (err: any) {
+ // didn't exist, don't worry
+ if (err.statusCode === 404) {
+ return
+ } else {
+ throw { ...err, status: err.statusCode }
+ }
+ }
+ }
+
+ async compact() {
+ const db = await this.checkSetup()
+ return this.updateOutput(() => db.compact())
+ }
+
+ // All below functions are in-frequently called, just utilise PouchDB
+ // for them as it implements them better than we can
+ async dump(stream: WriteStream, opts?: { filter?: any }) {
+ const pouch = getPouchDB(this.name)
+ // @ts-ignore
+ return pouch.dump(stream, opts)
+ }
+
+ async load(stream: ReadStream) {
+ const pouch = getPouchDB(this.name)
+ // @ts-ignore
+ return pouch.load(stream)
+ }
+
+ async createIndex(opts: DatabaseCreateIndexOpts) {
+ const pouch = getPouchDB(this.name)
+ return pouch.createIndex(opts)
+ }
+
+ async deleteIndex(opts: DatabaseDeleteIndexOpts) {
+ const pouch = getPouchDB(this.name)
+ return pouch.deleteIndex(opts)
+ }
+
+ async getIndexes() {
+ const pouch = getPouchDB(this.name)
+ return pouch.getIndexes()
+ }
+}
diff --git a/packages/backend-core/src/db/pouch.js b/packages/backend-core/src/db/couch/connections.ts
similarity index 55%
rename from packages/backend-core/src/db/pouch.js
rename to packages/backend-core/src/db/couch/connections.ts
index 12d7d787e3..a2206de634 100644
--- a/packages/backend-core/src/db/pouch.js
+++ b/packages/backend-core/src/db/couch/connections.ts
@@ -1,7 +1,39 @@
-const PouchDB = require("pouchdb")
-const env = require("../environment")
+import env from "../../environment"
-exports.getUrlInfo = (url = env.COUCH_DB_URL) => {
+export const getCouchInfo = () => {
+ const urlInfo = getUrlInfo()
+ let username
+ let password
+ if (env.COUCH_DB_USERNAME) {
+ // set from env
+ username = env.COUCH_DB_USERNAME
+ } else if (urlInfo.auth.username) {
+ // set from url
+ username = urlInfo.auth.username
+ } else if (!env.isTest()) {
+ throw new Error("CouchDB username not set")
+ }
+ if (env.COUCH_DB_PASSWORD) {
+ // set from env
+ password = env.COUCH_DB_PASSWORD
+ } else if (urlInfo.auth.password) {
+ // set from url
+ password = urlInfo.auth.password
+ } else if (!env.isTest()) {
+ throw new Error("CouchDB password not set")
+ }
+ const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+ return {
+ url: urlInfo.url!,
+ auth: {
+ username: username,
+ password: password,
+ },
+ cookie: `Basic ${authCookie}`,
+ }
+}
+
+export const getUrlInfo = (url = env.COUCH_DB_URL) => {
let cleanUrl, username, password, host
if (url) {
// Ensure the URL starts with a protocol
@@ -43,82 +75,3 @@ exports.getUrlInfo = (url = env.COUCH_DB_URL) => {
},
}
}
-
-exports.getCouchInfo = () => {
- const urlInfo = exports.getUrlInfo()
- let username
- let password
- if (env.COUCH_DB_USERNAME) {
- // set from env
- username = env.COUCH_DB_USERNAME
- } else if (urlInfo.auth.username) {
- // set from url
- username = urlInfo.auth.username
- } else if (!env.isTest()) {
- throw new Error("CouchDB username not set")
- }
- if (env.COUCH_DB_PASSWORD) {
- // set from env
- password = env.COUCH_DB_PASSWORD
- } else if (urlInfo.auth.password) {
- // set from url
- password = urlInfo.auth.password
- } else if (!env.isTest()) {
- throw new Error("CouchDB password not set")
- }
- const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
- return {
- url: urlInfo.url,
- auth: {
- username: username,
- password: password,
- },
- cookie: `Basic ${authCookie}`,
- }
-}
-
-/**
- * Return a constructor for PouchDB.
- * This should be rarely used outside of the main application config.
- * Exposed for exceptional cases such as in-memory views.
- */
-exports.getPouch = (opts = {}) => {
- let { url, cookie } = exports.getCouchInfo()
- let POUCH_DB_DEFAULTS = {
- prefix: url,
- fetch: (url, opts) => {
- // use a specific authorization cookie - be very explicit about how we authenticate
- opts.headers.set("Authorization", cookie)
- return PouchDB.fetch(url, opts)
- },
- }
-
- if (opts.inMemory) {
- const inMemory = require("pouchdb-adapter-memory")
- PouchDB.plugin(inMemory)
- POUCH_DB_DEFAULTS = {
- prefix: undefined,
- adapter: "memory",
- }
- }
-
- if (opts.onDisk) {
- POUCH_DB_DEFAULTS = {
- prefix: undefined,
- adapter: "leveldb",
- }
- }
-
- if (opts.replication) {
- const replicationStream = require("pouchdb-replication-stream")
- PouchDB.plugin(replicationStream.plugin)
- PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
- }
-
- if (opts.find) {
- const find = require("pouchdb-find")
- PouchDB.plugin(find)
- }
-
- return PouchDB.defaults(POUCH_DB_DEFAULTS)
-}
diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts
new file mode 100644
index 0000000000..c731d20d6c
--- /dev/null
+++ b/packages/backend-core/src/db/couch/index.ts
@@ -0,0 +1,4 @@
+export * from "./connections"
+export * from "./DatabaseImpl"
+export * from "./utils"
+export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts
new file mode 100644
index 0000000000..a6f4323d88
--- /dev/null
+++ b/packages/backend-core/src/db/couch/pouchDB.ts
@@ -0,0 +1,97 @@
+import PouchDB from "pouchdb"
+import env from "../../environment"
+import { PouchOptions } from "@budibase/types"
+import { getCouchInfo } from "./connections"
+
+let Pouch: any
+let initialised = false
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+export const getPouch = (opts: PouchOptions = {}) => {
+ let { url, cookie } = getCouchInfo()
+ let POUCH_DB_DEFAULTS = {
+ prefix: url,
+ fetch: (url: string, opts: any) => {
+ // use a specific authorization cookie - be very explicit about how we authenticate
+ opts.headers.set("Authorization", cookie)
+ return PouchDB.fetch(url, opts)
+ },
+ }
+
+ if (opts.inMemory) {
+ const inMemory = require("pouchdb-adapter-memory")
+ PouchDB.plugin(inMemory)
+ POUCH_DB_DEFAULTS = {
+ // @ts-ignore
+ adapter: "memory",
+ }
+ }
+
+ if (opts.onDisk) {
+ POUCH_DB_DEFAULTS = {
+ // @ts-ignore
+ adapter: "leveldb",
+ }
+ }
+
+ if (opts.replication) {
+ const replicationStream = require("pouchdb-replication-stream")
+ PouchDB.plugin(replicationStream.plugin)
+ // @ts-ignore
+ PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+ }
+
+ if (opts.find) {
+ const find = require("pouchdb-find")
+ PouchDB.plugin(find)
+ }
+
+ return PouchDB.defaults(POUCH_DB_DEFAULTS)
+}
+
+export function init(opts?: PouchOptions) {
+ Pouch = getPouch(opts)
+ initialised = true
+}
+
+const checkInitialised = () => {
+ if (!initialised) {
+ throw new Error("init has not been called")
+ }
+}
+
+export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
+ checkInitialised()
+ const db = new Pouch(dbName, opts)
+ const dbPut = db.put
+ db.put = async (doc: any, options = {}) => {
+ if (!doc.createdAt) {
+ doc.createdAt = new Date().toISOString()
+ }
+ doc.updatedAt = new Date().toISOString()
+ return dbPut(doc, options)
+ }
+ db.exists = async () => {
+ const info = await db.info()
+ return !info.error
+ }
+ return db
+}
+
+// use this function if you have called getPouchDB - close
+// the databases you've opened once finished
+export async function closePouchDB(db: PouchDB.Database) {
+ if (!db || env.isTest()) {
+ return
+ }
+ try {
+ // specifically await so that if there is an error, it can be ignored
+ return await db.close()
+ } catch (err) {
+ // ignore error, already closed
+ }
+}
diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts
new file mode 100644
index 0000000000..426bf92158
--- /dev/null
+++ b/packages/backend-core/src/db/couch/utils.ts
@@ -0,0 +1,36 @@
+import { getCouchInfo } from "./connections"
+import fetch from "node-fetch"
+import { checkSlashesInUrl } from "../../helpers"
+
+export async function directCouchCall(
+ path: string,
+ method: string = "GET",
+ body?: any
+) {
+ let { url, cookie } = getCouchInfo()
+ const couchUrl = `${url}/${path}`
+ const params: any = {
+ method: method,
+ headers: {
+ Authorization: cookie,
+ },
+ }
+ if (body && method !== "GET") {
+ params.body = JSON.stringify(body)
+ params.headers["Content-Type"] = "application/json"
+ }
+ return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
+}
+
+export async function directCouchQuery(
+ path: string,
+ method: string = "GET",
+ body?: any
+) {
+ const response = await directCouchCall(path, method, body)
+ if (response.status < 300) {
+ return await response.json()
+ } else {
+ throw "Cannot connect to CouchDB instance"
+ }
+}
diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts
new file mode 100644
index 0000000000..3887e8b09f
--- /dev/null
+++ b/packages/backend-core/src/db/db.ts
@@ -0,0 +1,46 @@
+import env from "../environment"
+import { directCouchQuery, getPouchDB } from "./couch"
+import { CouchFindOptions, Database } from "@budibase/types"
+import { DatabaseImpl } from "../db"
+
+const dbList = new Set()
+
+export function getDB(dbName?: string, opts?: any): Database {
+ // TODO: once using the test image, need to remove this
+ if (env.isTest()) {
+ dbList.add(dbName)
+ // @ts-ignore
+ return getPouchDB(dbName, opts)
+ }
+ return new DatabaseImpl(dbName, opts)
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+export async function doWithDB(dbName: string, cb: any, opts = {}) {
+ const db = getDB(dbName, opts)
+ // need this to be async so that we can correctly close DB after all
+ // async operations have been completed
+ return await cb(db)
+}
+
+export function allDbs() {
+ if (!env.isTest()) {
+ throw new Error("Cannot be used outside test environment.")
+ }
+ return [...dbList]
+}
+
+export async function directCouchAllDbs(queryString?: string) {
+ let couchPath = "/_all_dbs"
+ if (queryString) {
+ couchPath += `?${queryString}`
+ }
+ return await directCouchQuery(couchPath)
+}
+
+export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
+ const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
+ return { rows: json.docs, bookmark: json.bookmark }
+}
diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.js
deleted file mode 100644
index aa6f7ebc2c..0000000000
--- a/packages/backend-core/src/db/index.js
+++ /dev/null
@@ -1,91 +0,0 @@
-const pouch = require("./pouch")
-const env = require("../environment")
-
-const openDbs = []
-let PouchDB
-let initialised = false
-const dbList = new Set()
-
-if (env.MEMORY_LEAK_CHECK) {
- setInterval(() => {
- console.log("--- OPEN DBS ---")
- console.log(openDbs)
- }, 5000)
-}
-
-const put =
- dbPut =>
- async (doc, options = {}) => {
- if (!doc.createdAt) {
- doc.createdAt = new Date().toISOString()
- }
- doc.updatedAt = new Date().toISOString()
- return dbPut(doc, options)
- }
-
-const checkInitialised = () => {
- if (!initialised) {
- throw new Error("init has not been called")
- }
-}
-
-exports.init = opts => {
- PouchDB = pouch.getPouch(opts)
- initialised = true
-}
-
-// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
-// this function is prone to leaks, should only be used
-// in situations that using the function doWithDB does not work
-exports.dangerousGetDB = (dbName, opts) => {
- checkInitialised()
- if (env.isTest()) {
- dbList.add(dbName)
- }
- const db = new PouchDB(dbName, opts)
- if (env.MEMORY_LEAK_CHECK) {
- openDbs.push(db.name)
- }
- const dbPut = db.put
- db.put = put(dbPut)
- return db
-}
-
-// use this function if you have called dangerousGetDB - close
-// the databases you've opened once finished
-exports.closeDB = async db => {
- if (!db || env.isTest()) {
- return
- }
- if (env.MEMORY_LEAK_CHECK) {
- openDbs.splice(openDbs.indexOf(db.name), 1)
- }
- try {
- // specifically await so that if there is an error, it can be ignored
- return await db.close()
- } catch (err) {
- // ignore error, already closed
- }
-}
-
-// we have to use a callback for this so that we can close
-// the DB when we're done, without this manual requests would
-// need to close the database when done with it to avoid memory leaks
-exports.doWithDB = async (dbName, cb, opts = {}) => {
- const db = exports.dangerousGetDB(dbName, opts)
- // need this to be async so that we can correctly close DB after all
- // async operations have been completed
- try {
- return await cb(db)
- } finally {
- await exports.closeDB(db)
- }
-}
-
-exports.allDbs = () => {
- if (!env.isTest()) {
- throw new Error("Cannot be used outside test environment.")
- }
- checkInitialised()
- return [...dbList]
-}
diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts
new file mode 100644
index 0000000000..0d9f75fa18
--- /dev/null
+++ b/packages/backend-core/src/db/index.ts
@@ -0,0 +1,9 @@
+export * from "./couch"
+export * from "./db"
+export * from "./utils"
+export * from "./views"
+export * from "./conversions"
+export { default as Replication } from "./Replication"
+// exports to support old export structure
+export * from "../constants/db"
+export { getGlobalDBName, baseGlobalDBName } from "../context"
diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js
index bc0c638126..fc0094d354 100644
--- a/packages/backend-core/src/db/tests/index.spec.js
+++ b/packages/backend-core/src/db/tests/index.spec.js
@@ -1,11 +1,11 @@
-require("../../../tests/utilities/TestConfiguration")
-const { dangerousGetDB } = require("../")
+require("../../../tests")
+const { getDB } = require("../")
describe("db", () => {
describe("getDB", () => {
it("returns a db", async () => {
- const db = dangerousGetDB("test")
+ const db = getDB("test")
expect(db).toBeDefined()
expect(db._adapter).toBe("memory")
expect(db.prefix).toBe("_pouch_")
@@ -13,7 +13,7 @@ describe("db", () => {
})
it("uses the custom put function", async () => {
- const db = dangerousGetDB("test")
+ const db = getDB("test")
let doc = { _id: "test" }
await db.put(doc)
doc = await db.get(doc._id)
diff --git a/packages/backend-core/src/db/tests/pouch.spec.js b/packages/backend-core/src/db/tests/pouch.spec.js
index 30cdd0f5ec..f0abc82240 100644
--- a/packages/backend-core/src/db/tests/pouch.spec.js
+++ b/packages/backend-core/src/db/tests/pouch.spec.js
@@ -1,5 +1,5 @@
-require("../../../tests/utilities/TestConfiguration")
-const getUrlInfo = require("../pouch").getUrlInfo
+require("../../../tests")
+const getUrlInfo = require("../couch").getUrlInfo
describe("pouch", () => {
describe("Couch DB URL parsing", () => {
diff --git a/packages/backend-core/src/db/tests/utils.spec.js b/packages/backend-core/src/db/tests/utils.spec.js
index 5f9a224e7a..f95889c1cc 100644
--- a/packages/backend-core/src/db/tests/utils.spec.js
+++ b/packages/backend-core/src/db/tests/utils.spec.js
@@ -1,15 +1,17 @@
-require("../../../tests/utilities/TestConfiguration");
+require("../../../tests")
const {
- generateAppID,
getDevelopmentAppID,
getProdAppID,
isDevAppID,
isProdAppID,
+} = require("../conversions")
+const {
+ generateAppID,
getPlatformUrl,
getScopedConfig
} = require("../utils")
-const tenancy = require("../../tenancy");
-const { Configs, DEFAULT_TENANT_ID } = require("../../constants");
+const tenancy = require("../../tenancy")
+const { Config, DEFAULT_TENANT_ID } = require("../../constants")
const env = require("../../environment")
describe("utils", () => {
@@ -77,7 +79,7 @@ const setDbPlatformUrl = async () => {
const db = tenancy.getGlobalDB()
db.put({
_id: "config_settings",
- type: Configs.SETTINGS,
+ type: Config.SETTINGS,
config: {
platformUrl: DB_URL
}
@@ -178,7 +180,7 @@ describe("getScopedConfig", () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
await setDbPlatformUrl()
const db = tenancy.getGlobalDB()
- const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+ const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DB_URL)
})
})
@@ -186,7 +188,7 @@ describe("getScopedConfig", () => {
it("returns the platform url without an existing config", async () => {
await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
- const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+ const config = await getScopedConfig(db, { type: Config.SETTINGS })
expect(config.platformUrl).toBe(DEFAULT_URL)
})
})
diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts
index 8ab6fa6e98..590c3eeef8 100644
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@@ -1,26 +1,26 @@
-import { newid } from "../hashing"
-import { DEFAULT_TENANT_ID, Configs } from "../constants"
+import { newid } from "../newid"
import env from "../environment"
-import { SEPARATOR, DocumentTypes, UNICODE_MAX, ViewNames } from "./constants"
-import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
-import fetch from "node-fetch"
-import { doWithDB, allDbs } from "./index"
-import { getCouchInfo } from "./pouch"
+import {
+ DEFAULT_TENANT_ID,
+ SEPARATOR,
+ DocumentType,
+ UNICODE_MAX,
+ ViewName,
+ InternalTable,
+ APP_PREFIX,
+} from "../constants"
+import { getTenantId, getGlobalDB, getGlobalDBName } from "../context"
+import { doWithDB, allDbs, directCouchAllDbs } from "./db"
import { getAppMetadata } from "../cache/appMetadata"
-import { checkSlashesInUrl } from "../helpers"
import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
-import { APP_PREFIX } from "./constants"
import * as events from "../events"
-
-export * from "./constants"
-export * from "./conversions"
-export { default as Replication } from "./Replication"
+import { App, Database, ConfigType } from "@budibase/types"
/**
* Generates a new app ID.
* @returns {string} The new app ID which the app doc can be stored under.
*/
-export const generateAppID = (tenantId = null) => {
+export const generateAppID = (tenantId?: string | null) => {
let id = APP_PREFIX
if (tenantId) {
id += `${tenantId}${SEPARATOR}`
@@ -41,8 +41,8 @@ export const generateAppID = (tenantId = null) => {
* @returns {object} Parameters which can then be used with an allDocs request.
*/
export function getDocParams(
- docType: any,
- docId: any = null,
+ docType: string,
+ docId?: string | null,
otherProps: any = {}
) {
if (docId == null) {
@@ -55,19 +55,74 @@ export function getDocParams(
}
}
+/**
+ * Gets the DB allDocs/query params for retrieving a row.
+ * @param {string|null} tableId The table in which the rows have been stored.
+ * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
+ * left null to get all the rows in the table.
+ * @param {object} otherProps Any other properties to add to the request.
+ * @returns {object} Parameters which can then be used with an allDocs request.
+ */
+export function getRowParams(
+ tableId?: string | null,
+ rowId?: string | null,
+ otherProps = {}
+) {
+ if (tableId == null) {
+ return getDocParams(DocumentType.ROW, null, otherProps)
+ }
+
+ const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
+
+ return getDocParams(DocumentType.ROW, endOfKey, otherProps)
+}
+
/**
* Retrieve the correct index for a view based on default design DB.
*/
-export function getQueryIndex(viewName: ViewNames) {
+export function getQueryIndex(viewName: ViewName) {
return `database/${viewName}`
}
+/**
+ * Gets a new row ID for the specified table.
+ * @param {string} tableId The table which the row is being created for.
+ * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
+ * @returns {string} The new ID which a row doc can be stored under.
+ */
+export function generateRowID(tableId: string, id?: string) {
+ id = id || newid()
+ return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
+}
+
+/**
+ * Check if a given ID is that of a table.
+ * @returns {boolean}
+ */
+export const isTableId = (id: string) => {
+ // this includes datasource plus tables
+ return (
+ id &&
+ (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
+ id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+ )
+}
+
+/**
+ * Check if a given ID is that of a datasource or datasource plus.
+ * @returns {boolean}
+ */
+export const isDatasourceId = (id: string) => {
+ // this covers both datasources and datasource plus
+ return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
+}
+
/**
* Generates a new workspace ID.
* @returns {string} The new workspace ID which the workspace doc can be stored under.
*/
export function generateWorkspaceID() {
- return `${DocumentTypes.WORKSPACE}${SEPARATOR}${newid()}`
+ return `${DocumentType.WORKSPACE}${SEPARATOR}${newid()}`
}
/**
@@ -76,8 +131,8 @@ export function generateWorkspaceID() {
export function getWorkspaceParams(id = "", otherProps = {}) {
return {
...otherProps,
- startkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}`,
- endkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}${UNICODE_MAX}`,
+ startkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}`,
+ endkey: `${DocumentType.WORKSPACE}${SEPARATOR}${id}${UNICODE_MAX}`,
}
}
@@ -86,7 +141,7 @@ export function getWorkspaceParams(id = "", otherProps = {}) {
* @returns {string} The new user ID which the user doc can be stored under.
*/
export function generateGlobalUserID(id?: any) {
- return `${DocumentTypes.USER}${SEPARATOR}${id || newid()}`
+ return `${DocumentType.USER}${SEPARATOR}${id || newid()}`
}
/**
@@ -102,11 +157,38 @@ export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
// need to include this incase pagination
startkey: startkey
? startkey
- : `${DocumentTypes.USER}${SEPARATOR}${globalId}`,
- endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`,
+ : `${DocumentType.USER}${SEPARATOR}${globalId}`,
+ endkey: `${DocumentType.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`,
}
}
+/**
+ * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
+ */
+export function getUserMetadataParams(userId?: string | null, otherProps = {}) {
+ return getRowParams(InternalTable.USER_METADATA, userId, otherProps)
+}
+
+/**
+ * Generates a new user ID based on the passed in global ID.
+ * @param {string} globalId The ID of the global user.
+ * @returns {string} The new user ID which the user doc can be stored under.
+ */
+export function generateUserMetadataID(globalId: string) {
+ return generateRowID(InternalTable.USER_METADATA, globalId)
+}
+
+/**
+ * Breaks up the ID to get the global ID.
+ */
+export function getGlobalIDFromUserMetadataID(id: string) {
+ const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+ if (!id || !id.includes(prefix)) {
+ return id
+ }
+ return id.split(prefix)[1]
+}
+
export function getUsersByAppParams(appId: any, otherProps: any = {}) {
const prodAppId = getProdAppID(appId)
return {
@@ -121,7 +203,7 @@ export function getUsersByAppParams(appId: any, otherProps: any = {}) {
* @param ownerId The owner/user of the template, this could be global or a workspace level.
*/
export function generateTemplateID(ownerId: any) {
- return `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
+ return `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
}
export function generateAppUserID(prodAppId: string, userId: string) {
@@ -143,7 +225,7 @@ export function getTemplateParams(
if (templateId) {
final = templateId
} else {
- final = `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}`
+ final = `${DocumentType.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}`
}
return {
...otherProps,
@@ -156,20 +238,20 @@ export function getTemplateParams(
* Generates a new role ID.
* @returns {string} The new role ID which the role doc can be stored under.
*/
-export function generateRoleID(id: any) {
- return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
+export function generateRoleID(id?: any) {
+ return `${DocumentType.ROLE}${SEPARATOR}${id || newid()}`
}
/**
* Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
*/
-export function getRoleParams(roleId = null, otherProps = {}) {
- return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
+export function getRoleParams(roleId?: string | null, otherProps = {}) {
+ return getDocParams(DocumentType.ROLE, roleId, otherProps)
}
-export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) {
+export function getStartEndKeyURL(baseKey: any, tenantId?: string) {
const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
- return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
+ return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
}
/**
@@ -185,22 +267,10 @@ export async function getAllDbs(opts = { efficient: false }) {
return allDbs()
}
let dbs: any[] = []
- let { url, cookie } = getCouchInfo()
- async function addDbs(couchUrl: string) {
- const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
- method: "GET",
- headers: {
- Authorization: cookie,
- },
- })
- if (response.status === 200) {
- let json = await response.json()
- dbs = dbs.concat(json)
- } else {
- throw "Cannot connect to CouchDB instance"
- }
+ async function addDbs(queryString?: string) {
+ const json = await directCouchAllDbs(queryString)
+ dbs = dbs.concat(json)
}
- let couchUrl = `${url}/_all_dbs`
let tenantId = getTenantId()
if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
// just get all DBs when:
@@ -208,12 +278,12 @@ export async function getAllDbs(opts = { efficient: false }) {
// - default tenant
// - apps dbs don't contain tenant id
// - non-default tenant dbs are filtered out application side in getAllApps
- await addDbs(couchUrl)
+ await addDbs()
} else {
// get prod apps
- await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId))
+ await addDbs(getStartEndKeyURL(DocumentType.APP, tenantId))
// get dev apps
- await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId))
+ await addDbs(getStartEndKeyURL(DocumentType.APP_DEV, tenantId))
// add global db name
dbs.push(getGlobalDBName(tenantId))
}
@@ -226,21 +296,30 @@ export async function getAllDbs(opts = { efficient: false }) {
*
* @return {Promise} returns the app information document stored in each app database.
*/
-export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
+export async function getAllApps({
+ dev,
+ all,
+ idsOnly,
+ efficient,
+}: any = {}): Promise {
let tenantId = getTenantId()
if (!env.MULTI_TENANCY && !tenantId) {
tenantId = DEFAULT_TENANT_ID
}
let dbs = await getAllDbs({ efficient })
const appDbNames = dbs.filter((dbName: any) => {
+ if (env.isTest() && !dbName) {
+ return false
+ }
+
const split = dbName.split(SEPARATOR)
// it is an app, check the tenantId
- if (split[0] === DocumentTypes.APP) {
+ if (split[0] === DocumentType.APP) {
// tenantId is always right before the UUID
const possibleTenantId = split[split.length - 2]
const noTenantId =
- split.length === 2 || possibleTenantId === DocumentTypes.DEV
+ split.length === 2 || possibleTenantId === DocumentType.DEV
return (
(tenantId === DEFAULT_TENANT_ID && noTenantId) ||
@@ -250,7 +329,16 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
return false
})
if (idsOnly) {
- return appDbNames
+ const devAppIds = appDbNames.filter(appId => isDevAppID(appId))
+ const prodAppIds = appDbNames.filter(appId => !isDevAppID(appId))
+ switch (dev) {
+ case true:
+ return devAppIds
+ case false:
+ return prodAppIds
+ default:
+ return appDbNames
+ }
}
const appPromises = appDbNames.map((app: any) =>
// skip setup otherwise databases could be re-created
@@ -285,35 +373,23 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
* Utility function for getAllApps but filters to production apps only.
*/
export async function getProdAppIDs() {
- return (await getAllApps({ idsOnly: true })).filter(
- (id: any) => !isDevAppID(id)
- )
+ const apps = (await getAllApps({ idsOnly: true })) as string[]
+ return apps.filter((id: any) => !isDevAppID(id))
}
/**
* Utility function for the inverse of above.
*/
export async function getDevAppIDs() {
- return (await getAllApps({ idsOnly: true })).filter((id: any) =>
- isDevAppID(id)
- )
+ const apps = (await getAllApps({ idsOnly: true })) as string[]
+ return apps.filter((id: any) => isDevAppID(id))
}
export async function dbExists(dbName: any) {
- let exists = false
return doWithDB(
dbName,
- async (db: any) => {
- try {
- // check if database exists
- const info = await db.info()
- if (info && !info.error) {
- exists = true
- }
- } catch (err) {
- exists = false
- }
- return exists
+ async (db: Database) => {
+ return await db.exists()
},
{ skip_setup: true }
)
@@ -326,7 +402,7 @@ export async function dbExists(dbName: any) {
export const generateConfigID = ({ type, workspace, user }: any) => {
const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)
- return `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`
+ return `${DocumentType.CONFIG}${SEPARATOR}${scope}`
}
/**
@@ -340,8 +416,8 @@ export const getConfigParams = (
return {
...otherProps,
- startkey: `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`,
- endkey: `${DocumentTypes.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`,
+ startkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}`,
+ endkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`,
}
}
@@ -350,7 +426,22 @@ export const getConfigParams = (
* @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
*/
export const generateDevInfoID = (userId: any) => {
- return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}`
+ return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}`
+}
+
+/**
+ * Generates a new plugin ID - to be used in the global DB.
+ * @returns {string} The new plugin ID which a plugin metadata document can be stored under.
+ */
+export const generatePluginID = (name: string) => {
+ return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
+}
+
+/**
+ * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
+ */
+export const getPluginParams = (pluginId?: string | null, otherProps = {}) => {
+ return getDocParams(DocumentType.PLUGIN, pluginId, otherProps)
}
/**
@@ -397,7 +488,7 @@ export const getScopedFullConfig = async function (
)[0]
// custom logic for settings doc
- if (type === Configs.SETTINGS) {
+ if (type === ConfigType.SETTINGS) {
if (scopedConfig && scopedConfig.doc) {
// overrides affected by environment variables
scopedConfig.doc.config.platformUrl = await getPlatformUrl({
@@ -436,7 +527,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
// get the doc directly instead of with getScopedConfig to prevent loop
let settings
try {
- settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
+ settings = await db.get(generateConfigID({ type: ConfigType.SETTINGS }))
} catch (e: any) {
if (e.status !== 404) {
throw e
diff --git a/packages/backend-core/src/db/views.js b/packages/backend-core/src/db/views.js
deleted file mode 100644
index baf1807ca5..0000000000
--- a/packages/backend-core/src/db/views.js
+++ /dev/null
@@ -1,158 +0,0 @@
-const {
- DocumentTypes,
- ViewNames,
- DeprecatedViews,
- SEPARATOR,
-} = require("./utils")
-const { getGlobalDB } = require("../tenancy")
-
-const DESIGN_DB = "_design/database"
-
-function DesignDoc() {
- return {
- _id: DESIGN_DB,
- // view collation information, read before writing any complex views:
- // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification
- views: {},
- }
-}
-
-async function removeDeprecated(db, viewName) {
- if (!DeprecatedViews[viewName]) {
- return
- }
- try {
- const designDoc = await db.get(DESIGN_DB)
- for (let deprecatedNames of DeprecatedViews[viewName]) {
- delete designDoc.views[deprecatedNames]
- }
- await db.put(designDoc)
- } catch (err) {
- // doesn't exist, ignore
- }
-}
-
-exports.createNewUserEmailView = async () => {
- const db = getGlobalDB()
- let designDoc
- try {
- designDoc = await db.get(DESIGN_DB)
- } catch (err) {
- // no design doc, make one
- designDoc = DesignDoc()
- }
- const view = {
- // if using variables in a map function need to inject them before use
- map: `function(doc) {
- if (doc._id.startsWith("${DocumentTypes.USER}${SEPARATOR}")) {
- emit(doc.email.toLowerCase(), doc._id)
- }
- }`,
- }
- designDoc.views = {
- ...designDoc.views,
- [ViewNames.USER_BY_EMAIL]: view,
- }
- await db.put(designDoc)
-}
-
-exports.createUserAppView = async () => {
- const db = getGlobalDB()
- let designDoc
- try {
- designDoc = await db.get("_design/database")
- } catch (err) {
- // no design doc, make one
- designDoc = DesignDoc()
- }
- const view = {
- // if using variables in a map function need to inject them before use
- map: `function(doc) {
- if (doc._id.startsWith("${DocumentTypes.USER}${SEPARATOR}") && doc.roles) {
- for (let prodAppId of Object.keys(doc.roles)) {
- let emitted = prodAppId + "${SEPARATOR}" + doc._id
- emit(emitted, null)
- }
- }
- }`,
- }
- designDoc.views = {
- ...designDoc.views,
- [ViewNames.USER_BY_APP]: view,
- }
- await db.put(designDoc)
-}
-
-exports.createApiKeyView = async () => {
- const db = getGlobalDB()
- let designDoc
- try {
- designDoc = await db.get("_design/database")
- } catch (err) {
- designDoc = DesignDoc()
- }
- const view = {
- map: `function(doc) {
- if (doc._id.startsWith("${DocumentTypes.DEV_INFO}") && doc.apiKey) {
- emit(doc.apiKey, doc.userId)
- }
- }`,
- }
- designDoc.views = {
- ...designDoc.views,
- [ViewNames.BY_API_KEY]: view,
- }
- await db.put(designDoc)
-}
-
-exports.createUserBuildersView = async () => {
- const db = getGlobalDB()
- let designDoc
- try {
- designDoc = await db.get("_design/database")
- } catch (err) {
- // no design doc, make one
- designDoc = DesignDoc()
- }
- const view = {
- map: `function(doc) {
- if (doc.builder && doc.builder.global === true) {
- emit(doc._id, doc._id)
- }
- }`,
- }
- designDoc.views = {
- ...designDoc.views,
- [ViewNames.USER_BY_BUILDERS]: view,
- }
- await db.put(designDoc)
-}
-
-exports.queryGlobalView = async (viewName, params, db = null) => {
- const CreateFuncByName = {
- [ViewNames.USER_BY_EMAIL]: exports.createNewUserEmailView,
- [ViewNames.BY_API_KEY]: exports.createApiKeyView,
- [ViewNames.USER_BY_BUILDERS]: exports.createUserBuildersView,
- [ViewNames.USER_BY_APP]: exports.createUserAppView,
- }
- // can pass DB in if working with something specific
- if (!db) {
- db = getGlobalDB()
- }
- try {
- let response = (await db.query(`database/${viewName}`, params)).rows
- response = response.map(resp =>
- params.include_docs ? resp.doc : resp.value
- )
- return response.length <= 1 ? response[0] : response
- } catch (err) {
- if (err != null && err.name === "not_found") {
- const createFunc = CreateFuncByName[viewName]
- await removeDeprecated(db, viewName)
- await createFunc()
- return exports.queryGlobalView(viewName, params)
- } else {
- throw err
- }
- }
-}
diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts
new file mode 100644
index 0000000000..4a87be0a68
--- /dev/null
+++ b/packages/backend-core/src/db/views.ts
@@ -0,0 +1,199 @@
+import {
+ DocumentType,
+ ViewName,
+ DeprecatedViews,
+ SEPARATOR,
+ StaticDatabases,
+} from "../constants"
+import { getGlobalDB } from "../context"
+import { doWithDB } from "./"
+import { Database, DatabaseQueryOpts } from "@budibase/types"
+
+const DESIGN_DB = "_design/database"
+
+function DesignDoc() {
+ return {
+ _id: DESIGN_DB,
+ // view collation information, read before writing any complex views:
+ // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification
+ views: {},
+ }
+}
+
+interface DesignDocument {
+ views: any
+}
+
+async function removeDeprecated(db: Database, viewName: ViewName) {
+ // @ts-ignore
+ if (!DeprecatedViews[viewName]) {
+ return
+ }
+ try {
+ const designDoc = await db.get(DESIGN_DB)
+ // @ts-ignore
+ for (let deprecatedNames of DeprecatedViews[viewName]) {
+ delete designDoc.views[deprecatedNames]
+ }
+ await db.put(designDoc)
+ } catch (err) {
+ // doesn't exist, ignore
+ }
+}
+
+export async function createView(db: any, viewJs: string, viewName: string) {
+ let designDoc
+ try {
+ designDoc = (await db.get(DESIGN_DB)) as DesignDocument
+ } catch (err) {
+ // no design doc, make one
+ designDoc = DesignDoc()
+ }
+ const view = {
+ map: viewJs,
+ }
+ designDoc.views = {
+ ...designDoc.views,
+ [viewName]: view,
+ }
+ await db.put(designDoc)
+}
+
+export const createNewUserEmailView = async () => {
+ const db = getGlobalDB()
+ const viewJs = `function(doc) {
+ if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}")) {
+ emit(doc.email.toLowerCase(), doc._id)
+ }
+ }`
+ await createView(db, viewJs, ViewName.USER_BY_EMAIL)
+}
+
+export const createAccountEmailView = async () => {
+ const viewJs = `function(doc) {
+ if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+ emit(doc.email.toLowerCase(), doc._id)
+ }
+ }`
+ await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+ await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
+ })
+}
+
+export const createUserAppView = async () => {
+ const db = getGlobalDB()
+ const viewJs = `function(doc) {
+ if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
+ for (let prodAppId of Object.keys(doc.roles)) {
+ let emitted = prodAppId + "${SEPARATOR}" + doc._id
+ emit(emitted, null)
+ }
+ }
+ }`
+ await createView(db, viewJs, ViewName.USER_BY_APP)
+}
+
+export const createApiKeyView = async () => {
+ const db = getGlobalDB()
+ const viewJs = `function(doc) {
+ if (doc._id.startsWith("${DocumentType.DEV_INFO}") && doc.apiKey) {
+ emit(doc.apiKey, doc.userId)
+ }
+ }`
+ await createView(db, viewJs, ViewName.BY_API_KEY)
+}
+
+export const createUserBuildersView = async () => {
+ const db = getGlobalDB()
+ const viewJs = `function(doc) {
+ if (doc.builder && doc.builder.global === true) {
+ emit(doc._id, doc._id)
+ }
+ }`
+ await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
+}
+
+export const createPlatformUserView = async () => {
+ const viewJs = `function(doc) {
+ if (doc.tenantId) {
+ emit(doc._id.toLowerCase(), doc._id)
+ }
+ }`
+ await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+ await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
+ })
+}
+
+export interface QueryViewOptions {
+ arrayResponse?: boolean
+}
+
+export const queryView = async (
+ viewName: ViewName,
+ params: DatabaseQueryOpts,
+ db: Database,
+ createFunc: any,
+ opts?: QueryViewOptions
+): Promise => {
+ try {
+ let response = await db.query(`database/${viewName}`, params)
+ const rows = response.rows
+ const docs = rows.map((row: any) =>
+ params.include_docs ? row.doc : row.value
+ )
+
+ // if arrayResponse has been requested, always return array regardless of length
+ if (opts?.arrayResponse) {
+ return docs as T[]
+ } else {
+ // return the single document if there is only one
+ return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
+ }
+ } catch (err: any) {
+ const pouchNotFound = err && err.name === "not_found"
+ const couchNotFound = err && err.status === 404
+ if (pouchNotFound || couchNotFound) {
+ await removeDeprecated(db, viewName)
+ await createFunc()
+ return queryView(viewName, params, db, createFunc, opts)
+ } else {
+ throw err
+ }
+ }
+}
+
+export const queryPlatformView = async (
+ viewName: ViewName,
+ params: DatabaseQueryOpts,
+ opts?: QueryViewOptions
+): Promise => {
+ const CreateFuncByName: any = {
+ [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+ [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
+ }
+
+ return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+ const createFn = CreateFuncByName[viewName]
+ return queryView(viewName, params, db, createFn, opts)
+ })
+}
+
+export const queryGlobalView = async (
+ viewName: ViewName,
+ params: DatabaseQueryOpts,
+ db?: Database,
+ opts?: QueryViewOptions
+): Promise => {
+ const CreateFuncByName: any = {
+ [ViewName.USER_BY_EMAIL]: createNewUserEmailView,
+ [ViewName.BY_API_KEY]: createApiKeyView,
+ [ViewName.USER_BY_BUILDERS]: createUserBuildersView,
+ [ViewName.USER_BY_APP]: createUserAppView,
+ }
+ // can pass DB in if working with something specific
+ if (!db) {
+ db = getGlobalDB()
+ }
+ const createFn = CreateFuncByName[viewName]
+ return queryView(viewName, params, db!, createFn, opts)
+}
diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 0348d921ab..51ab101b3c 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -16,9 +16,19 @@ if (!LOADED && isDev() && !isTest()) {
LOADED = true
}
-const env = {
+const DefaultBucketName = {
+ BACKUPS: "backups",
+ APPS: "prod-budi-app-assets",
+ TEMPLATES: "templates",
+ GLOBAL: "global",
+ CLOUD: "prod-budi-tenant-uploads",
+ PLUGINS: "plugins",
+}
+
+const environment = {
isTest,
isDev,
+ JS_BCRYPT: process.env.JS_BCRYPT,
JWT_SECRET: process.env.JWT_SECRET,
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
@@ -36,7 +46,7 @@ const env = {
MULTI_TENANCY: process.env.MULTI_TENANCY,
ACCOUNT_PORTAL_URL:
process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
- ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
+ ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "",
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
@@ -44,34 +54,39 @@ const env = {
POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
- BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
- APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
- TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates",
- GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global",
+ BACKUPS_BUCKET_NAME:
+ process.env.BACKUPS_BUCKET_NAME || DefaultBucketName.BACKUPS,
+ APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || DefaultBucketName.APPS,
+ TEMPLATES_BUCKET_NAME:
+ process.env.TEMPLATES_BUCKET_NAME || DefaultBucketName.TEMPLATES,
+ GLOBAL_BUCKET_NAME:
+ process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
GLOBAL_CLOUD_BUCKET_NAME:
- process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
+ process.env.GLOBAL_CLOUD_BUCKET_NAME || DefaultBucketName.CLOUD,
+ PLUGIN_BUCKET_NAME:
+ process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
USE_COUCH: process.env.USE_COUCH || true,
DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
SERVICE: process.env.SERVICE || "budibase",
- MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
LOG_LEVEL: process.env.LOG_LEVEL,
SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
DEPLOYMENT_ENVIRONMENT:
process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
_set(key: any, value: any) {
process.env[key] = value
- module.exports[key] = value
+ // @ts-ignore
+ environment[key] = value
},
}
// clean up any environment variable edge cases
-for (let [key, value] of Object.entries(env)) {
+for (let [key, value] of Object.entries(environment)) {
// handle the edge case of "0" to disable an environment variable
if (value === "0") {
// @ts-ignore
- env[key] = 0
+ environment[key] = 0
}
}
-export = env
+export = environment
diff --git a/packages/backend-core/src/errors/base.js b/packages/backend-core/src/errors/base.js
deleted file mode 100644
index 7cb0c0fc23..0000000000
--- a/packages/backend-core/src/errors/base.js
+++ /dev/null
@@ -1,11 +0,0 @@
-class BudibaseError extends Error {
- constructor(message, code, type) {
- super(message)
- this.code = code
- this.type = type
- }
-}
-
-module.exports = {
- BudibaseError,
-}
diff --git a/packages/backend-core/src/errors/base.ts b/packages/backend-core/src/errors/base.ts
new file mode 100644
index 0000000000..801dcf168d
--- /dev/null
+++ b/packages/backend-core/src/errors/base.ts
@@ -0,0 +1,10 @@
+export class BudibaseError extends Error {
+ code: string
+ type: string
+
+ constructor(message: string, code: string, type: string) {
+ super(message)
+ this.code = code
+ this.type = type
+ }
+}
diff --git a/packages/backend-core/src/errors/generic.js b/packages/backend-core/src/errors/generic.js
deleted file mode 100644
index 5c7661f035..0000000000
--- a/packages/backend-core/src/errors/generic.js
+++ /dev/null
@@ -1,11 +0,0 @@
-const { BudibaseError } = require("./base")
-
-class GenericError extends BudibaseError {
- constructor(message, code, type) {
- super(message, code, type ? type : "generic")
- }
-}
-
-module.exports = {
- GenericError,
-}
diff --git a/packages/backend-core/src/errors/generic.ts b/packages/backend-core/src/errors/generic.ts
new file mode 100644
index 0000000000..71b3352438
--- /dev/null
+++ b/packages/backend-core/src/errors/generic.ts
@@ -0,0 +1,7 @@
+import { BudibaseError } from "./base"
+
+export class GenericError extends BudibaseError {
+ constructor(message: string, code: string, type: string) {
+ super(message, code, type ? type : "generic")
+ }
+}
diff --git a/packages/backend-core/src/errors/http.js b/packages/backend-core/src/errors/http.js
deleted file mode 100644
index 8e7cab4638..0000000000
--- a/packages/backend-core/src/errors/http.js
+++ /dev/null
@@ -1,12 +0,0 @@
-const { GenericError } = require("./generic")
-
-class HTTPError extends GenericError {
- constructor(message, httpStatus, code = "http", type = "generic") {
- super(message, code, type)
- this.status = httpStatus
- }
-}
-
-module.exports = {
- HTTPError,
-}
diff --git a/packages/backend-core/src/errors/http.ts b/packages/backend-core/src/errors/http.ts
new file mode 100644
index 0000000000..182e009f58
--- /dev/null
+++ b/packages/backend-core/src/errors/http.ts
@@ -0,0 +1,15 @@
+import { GenericError } from "./generic"
+
+export class HTTPError extends GenericError {
+ status: number
+
+ constructor(
+ message: string,
+ httpStatus: number,
+ code = "http",
+ type = "generic"
+ ) {
+ super(message, code, type)
+ this.status = httpStatus
+ }
+}
diff --git a/packages/backend-core/src/errors/index.js b/packages/backend-core/src/errors/index.ts
similarity index 65%
rename from packages/backend-core/src/errors/index.js
rename to packages/backend-core/src/errors/index.ts
index 31ffd739a0..be6657093d 100644
--- a/packages/backend-core/src/errors/index.js
+++ b/packages/backend-core/src/errors/index.ts
@@ -1,5 +1,6 @@
-const http = require("./http")
-const licensing = require("./licensing")
+import { HTTPError } from "./http"
+import { UsageLimitError, FeatureDisabledError } from "./licensing"
+import * as licensing from "./licensing"
const codes = {
...licensing.codes,
@@ -11,7 +12,7 @@ const context = {
...licensing.context,
}
-const getPublicError = err => {
+const getPublicError = (err: any) => {
let error
if (err.code || err.type) {
// add generic error information
@@ -32,13 +33,15 @@ const getPublicError = err => {
return error
}
-module.exports = {
+const pkg = {
codes,
types,
errors: {
- UsageLimitError: licensing.UsageLimitError,
- FeatureDisabledError: licensing.FeatureDisabledError,
- HTTPError: http.HTTPError,
+ UsageLimitError,
+ FeatureDisabledError,
+ HTTPError,
},
getPublicError,
}
+
+export = pkg
diff --git a/packages/backend-core/src/errors/licensing.js b/packages/backend-core/src/errors/licensing.js
deleted file mode 100644
index 85d207ac35..0000000000
--- a/packages/backend-core/src/errors/licensing.js
+++ /dev/null
@@ -1,43 +0,0 @@
-const { HTTPError } = require("./http")
-
-const type = "license_error"
-
-const codes = {
- USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded",
- FEATURE_DISABLED: "feature_disabled",
-}
-
-const context = {
- [codes.USAGE_LIMIT_EXCEEDED]: err => {
- return {
- limitName: err.limitName,
- }
- },
- [codes.FEATURE_DISABLED]: err => {
- return {
- featureName: err.featureName,
- }
- },
-}
-
-class UsageLimitError extends HTTPError {
- constructor(message, limitName) {
- super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type)
- this.limitName = limitName
- }
-}
-
-class FeatureDisabledError extends HTTPError {
- constructor(message, featureName) {
- super(message, 400, codes.FEATURE_DISABLED, type)
- this.featureName = featureName
- }
-}
-
-module.exports = {
- type,
- codes,
- context,
- UsageLimitError,
- FeatureDisabledError,
-}
diff --git a/packages/backend-core/src/errors/licensing.ts b/packages/backend-core/src/errors/licensing.ts
new file mode 100644
index 0000000000..7ffcefa167
--- /dev/null
+++ b/packages/backend-core/src/errors/licensing.ts
@@ -0,0 +1,39 @@
+import { HTTPError } from "./http"
+
+export const type = "license_error"
+
+export const codes = {
+ USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded",
+ FEATURE_DISABLED: "feature_disabled",
+}
+
+export const context = {
+ [codes.USAGE_LIMIT_EXCEEDED]: (err: any) => {
+ return {
+ limitName: err.limitName,
+ }
+ },
+ [codes.FEATURE_DISABLED]: (err: any) => {
+ return {
+ featureName: err.featureName,
+ }
+ },
+}
+
+export class UsageLimitError extends HTTPError {
+ limitName: string
+
+ constructor(message: string, limitName: string) {
+ super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type)
+ this.limitName = limitName
+ }
+}
+
+export class FeatureDisabledError extends HTTPError {
+ featureName: string
+
+ constructor(message: string, featureName: string) {
+ super(message, 400, codes.FEATURE_DISABLED, type)
+ this.featureName = featureName
+ }
+}
diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts
index 802b6d6314..f621a9c98b 100644
--- a/packages/backend-core/src/events/analytics.ts
+++ b/packages/backend-core/src/events/analytics.ts
@@ -1,8 +1,8 @@
import env from "../environment"
-import tenancy from "../tenancy"
+import * as tenancy from "../tenancy"
import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
-import { withCache, TTL, CacheKeys } from "../cache/generic"
+import { Config } from "../constants"
+import { withCache, TTL, CacheKey } from "../cache"
export const enabled = async () => {
// cloud - always use the environment variable
@@ -13,7 +13,7 @@ export const enabled = async () => {
// self host - prefer the settings doc
// use cache as events have high throughput
const enabledInDB = await withCache(
- CacheKeys.ANALYTICS_ENABLED,
+ CacheKey.ANALYTICS_ENABLED,
TTL.ONE_DAY,
async () => {
const settings = await getSettingsDoc()
@@ -45,9 +45,7 @@ const getSettingsDoc = async () => {
const db = tenancy.getGlobalDB()
let settings
try {
- settings = await db.get(
- dbUtils.generateConfigID({ type: Configs.SETTINGS })
- )
+ settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))
} catch (e: any) {
if (e.status !== 404) {
throw e
diff --git a/packages/backend-core/src/events/backfill.ts b/packages/backend-core/src/events/backfill.ts
index e4577c5ab4..c8025a8e4e 100644
--- a/packages/backend-core/src/events/backfill.ts
+++ b/packages/backend-core/src/events/backfill.ts
@@ -21,7 +21,7 @@ import {
AppCreatedEvent,
} from "@budibase/types"
import * as context from "../context"
-import { CacheKeys } from "../cache/generic"
+import { CacheKey } from "../cache/generic"
import * as cache from "../cache/generic"
// LIFECYCLE
@@ -48,18 +48,18 @@ export const end = async () => {
// CRUD
const getBackfillMetadata = async (): Promise => {
- return cache.get(CacheKeys.BACKFILL_METADATA)
+ return cache.get(CacheKey.BACKFILL_METADATA)
}
const saveBackfillMetadata = async (
backfill: BackfillMetadata
): Promise => {
// no TTL - deleted by backfill
- return cache.store(CacheKeys.BACKFILL_METADATA, backfill)
+ return cache.store(CacheKey.BACKFILL_METADATA, backfill)
}
const deleteBackfillMetadata = async (): Promise => {
- await cache.delete(CacheKeys.BACKFILL_METADATA)
+ await cache.destroy(CacheKey.BACKFILL_METADATA)
}
const clearEvents = async () => {
@@ -70,7 +70,7 @@ const clearEvents = async () => {
for (const key of keys) {
// delete each key
// don't use tenancy, already in the key
- await cache.delete(key, { useTenancy: false })
+ await cache.destroy(key, { useTenancy: false })
}
}
@@ -167,7 +167,7 @@ const getEventKey = (event?: Event, properties?: any) => {
const tenantId = context.getTenantId()
if (event) {
- eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}`
+ eventKey = `${CacheKey.EVENTS}:${tenantId}:${event}`
// use some properties to make the key more unique
const custom = CUSTOM_PROPERTY_SUFFIX[event]
@@ -176,7 +176,7 @@ const getEventKey = (event?: Event, properties?: any) => {
eventKey = `${eventKey}:${suffix}`
}
} else {
- eventKey = `${CacheKeys.EVENTS}:${tenantId}:*`
+ eventKey = `${CacheKey.EVENTS}:${tenantId}:*`
}
return eventKey
diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts
index a29a6821cd..b93bd44968 100644
--- a/packages/backend-core/src/events/identification.ts
+++ b/packages/backend-core/src/events/identification.ts
@@ -19,10 +19,10 @@ import {
} from "@budibase/types"
import { processors } from "./processors"
import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
-import * as hashing from "../hashing"
+import { Config } from "../constants"
+import { newid } from "../utils"
import * as installation from "../installation"
-import { withCache, TTL, CacheKeys } from "../cache/generic"
+import { withCache, TTL, CacheKey } from "../cache/generic"
const pkg = require("../../package.json")
@@ -270,17 +270,17 @@ const getEventTenantId = async (tenantId: string): Promise => {
const getUniqueTenantId = async (tenantId: string): Promise => {
// make sure this tenantId always matches the tenantId in context
return context.doInTenant(tenantId, () => {
- return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
+ return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
const db = context.getGlobalDB()
const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
- type: Configs.SETTINGS,
+ type: Config.SETTINGS,
})
let uniqueTenantId: string
if (config.config.uniqueTenantId) {
return config.config.uniqueTenantId
} else {
- uniqueTenantId = `${hashing.newid()}_${tenantId}`
+ uniqueTenantId = `${newid()}_${tenantId}`
config.config.uniqueTenantId = uniqueTenantId
await db.put(config)
return uniqueTenantId
diff --git a/packages/backend-core/src/events/index.ts b/packages/backend-core/src/events/index.ts
index 814399655d..f94c8b0267 100644
--- a/packages/backend-core/src/events/index.ts
+++ b/packages/backend-core/src/events/index.ts
@@ -8,4 +8,5 @@ import { processors } from "./processors"
export const shutdown = () => {
processors.shutdown()
+ console.log("Events shutdown")
}
diff --git a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
index 2ee7a02afa..f9d7547120 100644
--- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
+++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
@@ -2,7 +2,7 @@ import { Event, Identity, Group, IdentityType } from "@budibase/types"
import { EventProcessor } from "./types"
import env from "../../environment"
import * as analytics from "../analytics"
-import PosthogProcessor from "./PosthogProcessor"
+import PosthogProcessor from "./posthog"
/**
* Events that are always captured.
@@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor {
return
}
if (this.posthog) {
- this.posthog.processEvent(event, identity, properties, timestamp)
+ await this.posthog.processEvent(event, identity, properties, timestamp)
}
}
@@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor {
return
}
if (this.posthog) {
- this.posthog.identify(identity, timestamp)
+ await this.posthog.identify(identity, timestamp)
}
}
async identifyGroup(group: Group, timestamp?: string | number) {
// Group indentifications (tenant and installation) always on
if (this.posthog) {
- this.posthog.identifyGroup(group, timestamp)
+ await this.posthog.identifyGroup(group, timestamp)
}
}
diff --git a/packages/backend-core/src/events/processors/LoggingProcessor.ts b/packages/backend-core/src/events/processors/LoggingProcessor.ts
index a517fba09c..d41a82fbb4 100644
--- a/packages/backend-core/src/events/processors/LoggingProcessor.ts
+++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts
@@ -23,9 +23,11 @@ export default class LoggingProcessor implements EventProcessor {
return
}
let timestampString = getTimestampString(timestamp)
- console.log(
- `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
- )
+ let message = `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+ if (env.isDev()) {
+ message = message + `[debug: [properties=${JSON.stringify(properties)}] ]`
+ }
+ console.log(message)
}
async identify(identity: Identity, timestamp?: string | number) {
diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts
similarity index 89%
rename from packages/backend-core/src/events/processors/PosthogProcessor.ts
rename to packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts
index 9d68d3919a..593e5ff082 100644
--- a/packages/backend-core/src/events/processors/PosthogProcessor.ts
+++ b/packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts
@@ -1,9 +1,10 @@
import PostHog from "posthog-node"
import { Event, Identity, Group, BaseEvent } from "@budibase/types"
-import { EventProcessor } from "./types"
-import env from "../../environment"
-import * as context from "../../context"
-const pkg = require("../../../package.json")
+import { EventProcessor } from "../types"
+import env from "../../../environment"
+import * as context from "../../../context"
+import * as rateLimiting from "./rateLimiting"
+const pkg = require("../../../../package.json")
const EXCLUDED_EVENTS: Event[] = [
Event.USER_UPDATED,
@@ -42,6 +43,10 @@ export default class PosthogProcessor implements EventProcessor {
return
}
+ if (await rateLimiting.limited(event)) {
+ return
+ }
+
properties.version = pkg.version
properties.service = env.SERVICE
properties.environment = identity.environment
diff --git a/packages/backend-core/src/events/processors/posthog/index.ts b/packages/backend-core/src/events/processors/posthog/index.ts
new file mode 100644
index 0000000000..dceb10d2cd
--- /dev/null
+++ b/packages/backend-core/src/events/processors/posthog/index.ts
@@ -0,0 +1,2 @@
+import PosthogProcessor from "./PosthogProcessor"
+export default PosthogProcessor
diff --git a/packages/backend-core/src/events/processors/posthog/rateLimiting.ts b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts
new file mode 100644
index 0000000000..89da10defa
--- /dev/null
+++ b/packages/backend-core/src/events/processors/posthog/rateLimiting.ts
@@ -0,0 +1,106 @@
+import { Event } from "@budibase/types"
+import { CacheKey, TTL } from "../../../cache/generic"
+import * as cache from "../../../cache/generic"
+import * as context from "../../../context"
+
+type RateLimitedEvent =
+ | Event.SERVED_BUILDER
+ | Event.SERVED_APP_PREVIEW
+ | Event.SERVED_APP
+
+const isRateLimited = (event: Event): event is RateLimitedEvent => {
+ return (
+ event === Event.SERVED_BUILDER ||
+ event === Event.SERVED_APP_PREVIEW ||
+ event === Event.SERVED_APP
+ )
+}
+
+const isPerApp = (event: RateLimitedEvent) => {
+ return event === Event.SERVED_APP_PREVIEW || event === Event.SERVED_APP
+}
+
+interface EventProperties {
+ timestamp: number
+}
+
+enum RateLimit {
+ CALENDAR_DAY = "calendarDay",
+}
+
+const RATE_LIMITS = {
+ [Event.SERVED_APP]: RateLimit.CALENDAR_DAY,
+ [Event.SERVED_APP_PREVIEW]: RateLimit.CALENDAR_DAY,
+ [Event.SERVED_BUILDER]: RateLimit.CALENDAR_DAY,
+}
+
+/**
+ * Check if this event should be sent right now
+ * Return false to signal the event SHOULD be sent
+ * Return true to signal the event should NOT be sent
+ */
+export const limited = async (event: Event): Promise => {
+ // not a rate limited event -- send
+ if (!isRateLimited(event)) {
+ return false
+ }
+
+ const cachedEvent = await readEvent(event)
+ if (cachedEvent) {
+ const timestamp = new Date(cachedEvent.timestamp)
+ const limit = RATE_LIMITS[event]
+ switch (limit) {
+ case RateLimit.CALENDAR_DAY: {
+ // get midnight at the start of the next day for the timestamp
+ timestamp.setDate(timestamp.getDate() + 1)
+ timestamp.setHours(0, 0, 0, 0)
+
+ // if we have passed the threshold into the next day
+ if (Date.now() > timestamp.getTime()) {
+ // update the timestamp in the event -- send
+ await recordEvent(event, { timestamp: Date.now() })
+ return false
+ } else {
+ // still within the limited period -- don't send
+ return true
+ }
+ }
+ }
+ } else {
+ // no event present i.e. expired -- send
+ await recordEvent(event, { timestamp: Date.now() })
+ return false
+ }
+}
+
+const eventKey = (event: RateLimitedEvent) => {
+ let key = `${CacheKey.EVENTS_RATE_LIMIT}:${event}`
+ if (isPerApp(event)) {
+ key = key + ":" + context.getAppId()
+ }
+ return key
+}
+
+const readEvent = async (
+ event: RateLimitedEvent
+): Promise => {
+ const key = eventKey(event)
+ const result = await cache.get(key)
+ return result as EventProperties
+}
+
+const recordEvent = async (
+ event: RateLimitedEvent,
+ properties: EventProperties
+) => {
+ const key = eventKey(event)
+ const limit = RATE_LIMITS[event]
+ let ttl
+ switch (limit) {
+ case RateLimit.CALENDAR_DAY: {
+ ttl = TTL.ONE_DAY
+ }
+ }
+
+ await cache.store(key, properties, ttl)
+}
diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts
new file mode 100644
index 0000000000..349a0427ac
--- /dev/null
+++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts
@@ -0,0 +1,145 @@
+import "../../../../../tests"
+import PosthogProcessor from "../PosthogProcessor"
+import { Event, IdentityType, Hosting } from "@budibase/types"
+const tk = require("timekeeper")
+import * as cache from "../../../../cache/generic"
+import { CacheKey } from "../../../../cache/generic"
+import * as context from "../../../../context"
+
+const newIdentity = () => {
+ return {
+ id: "test",
+ type: IdentityType.USER,
+ hosting: Hosting.SELF,
+ environment: "test",
+ }
+}
+
+describe("PosthogProcessor", () => {
+ beforeEach(async () => {
+ jest.clearAllMocks()
+ await cache.bustCache(
+ `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
+ )
+ })
+
+ describe("processEvent", () => {
+ it("processes event", async () => {
+ const processor = new PosthogProcessor("test")
+
+ const identity = newIdentity()
+ const properties = {}
+
+ await processor.processEvent(Event.APP_CREATED, identity, properties)
+
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+ })
+
+ it("honours exclusions", async () => {
+ const processor = new PosthogProcessor("test")
+
+ const identity = newIdentity()
+ const properties = {}
+
+ await processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties)
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
+ })
+
+ describe("rate limiting", () => {
+ it("sends daily event once in same day", async () => {
+ const processor = new PosthogProcessor("test")
+ const identity = newIdentity()
+ const properties = {}
+
+ tk.freeze(new Date(2022, 0, 1, 14, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+ // go forward one hour
+ tk.freeze(new Date(2022, 0, 1, 15, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
+ })
+
+ it("sends daily event once per unique day", async () => {
+ const processor = new PosthogProcessor("test")
+ const identity = newIdentity()
+ const properties = {}
+
+ tk.freeze(new Date(2022, 0, 1, 14, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+ // go forward into next day
+ tk.freeze(new Date(2022, 0, 2, 9, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+ // go forward into next day
+ tk.freeze(new Date(2022, 0, 3, 5, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+ // go forward one hour
+ tk.freeze(new Date(2022, 0, 3, 6, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(3)
+ })
+
+ it("sends event again after cache expires", async () => {
+ const processor = new PosthogProcessor("test")
+ const identity = newIdentity()
+ const properties = {}
+
+ tk.freeze(new Date(2022, 0, 1, 14, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+ await cache.bustCache(
+ `${CacheKey.EVENTS_RATE_LIMIT}:${Event.SERVED_BUILDER}`
+ )
+
+ tk.freeze(new Date(2022, 0, 1, 14, 0))
+ await processor.processEvent(Event.SERVED_BUILDER, identity, properties)
+
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(2)
+ })
+
+ it("sends per app events once per day per app", async () => {
+ const processor = new PosthogProcessor("test")
+ const identity = newIdentity()
+ const properties = {}
+
+ const runAppEvents = async (appId: string) => {
+ await context.doInAppContext(appId, async () => {
+ tk.freeze(new Date(2022, 0, 1, 14, 0))
+ await processor.processEvent(Event.SERVED_APP, identity, properties)
+ await processor.processEvent(
+ Event.SERVED_APP_PREVIEW,
+ identity,
+ properties
+ )
+
+ // go forward one hour - should be ignored
+ tk.freeze(new Date(2022, 0, 1, 15, 0))
+ await processor.processEvent(Event.SERVED_APP, identity, properties)
+ await processor.processEvent(
+ Event.SERVED_APP_PREVIEW,
+ identity,
+ properties
+ )
+
+ // go forward into next day
+ tk.freeze(new Date(2022, 0, 2, 9, 0))
+
+ await processor.processEvent(Event.SERVED_APP, identity, properties)
+ await processor.processEvent(
+ Event.SERVED_APP_PREVIEW,
+ identity,
+ properties
+ )
+ })
+ }
+
+ await runAppEvents("app_1")
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(4)
+
+ await runAppEvents("app_2")
+ expect(processor.posthog.capture).toHaveBeenCalledTimes(8)
+ })
+ })
+ })
+})
diff --git a/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts
deleted file mode 100644
index 4a6d55b272..0000000000
--- a/packages/backend-core/src/events/processors/tests/PosthogProcessor.spec.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import PosthogProcessor from "../PosthogProcessor"
-import { Event, IdentityType, Hosting } from "@budibase/types"
-
-const newIdentity = () => {
- return {
- id: "test",
- type: IdentityType.USER,
- hosting: Hosting.SELF,
- environment: "test",
- }
-}
-
-describe("PosthogProcessor", () => {
- beforeEach(() => {
- jest.clearAllMocks()
- })
-
- describe("processEvent", () => {
- it("processes event", () => {
- const processor = new PosthogProcessor("test")
-
- const identity = newIdentity()
- const properties = {}
-
- processor.processEvent(Event.APP_CREATED, identity, properties)
-
- expect(processor.posthog.capture).toHaveBeenCalledTimes(1)
- })
-
- it("honours exclusions", () => {
- const processor = new PosthogProcessor("test")
-
- const identity = newIdentity()
- const properties = {}
-
- processor.processEvent(Event.AUTH_SSO_UPDATED, identity, properties)
- expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
- })
- })
-})
diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts
index 95f9cb8db6..8b2574b739 100644
--- a/packages/backend-core/src/events/publishers/automation.ts
+++ b/packages/backend-core/src/events/publishers/automation.ts
@@ -72,7 +72,7 @@ export async function stepCreated(
automationId: automation._id as string,
triggerId: automation.definition?.trigger?.id,
triggerType: automation.definition?.trigger?.stepId,
- stepId: step.id,
+ stepId: step.id!,
stepType: step.stepId,
}
await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
@@ -87,7 +87,7 @@ export async function stepDeleted(
automationId: automation._id as string,
triggerId: automation.definition?.trigger?.id,
triggerType: automation.definition?.trigger?.stepId,
- stepId: step.id,
+ stepId: step.id!,
stepType: step.stepId,
}
await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
diff --git a/packages/backend-core/src/events/publishers/backup.ts b/packages/backend-core/src/events/publishers/backup.ts
new file mode 100644
index 0000000000..4a68364016
--- /dev/null
+++ b/packages/backend-core/src/events/publishers/backup.ts
@@ -0,0 +1,34 @@
+import {
+ AppBackup,
+ AppBackupRestoreEvent,
+ AppBackupTriggeredEvent,
+ AppBackupTrigger,
+ AppBackupType,
+ Event,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+export async function appBackupRestored(backup: AppBackup) {
+ const properties: AppBackupRestoreEvent = {
+ appId: backup.appId,
+ restoreId: backup._id!,
+ backupCreatedAt: backup.timestamp,
+ }
+
+ await publishEvent(Event.APP_BACKUP_RESTORED, properties)
+}
+
+export async function appBackupTriggered(
+ appId: string,
+ backupId: string,
+ type: AppBackupType,
+ trigger: AppBackupTrigger
+) {
+ const properties: AppBackupTriggeredEvent = {
+ appId: appId,
+ backupId,
+ type,
+ trigger,
+ }
+ await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
+}
diff --git a/packages/backend-core/src/events/publishers/datasource.ts b/packages/backend-core/src/events/publishers/datasource.ts
index 3cd68033fc..d3ea7402f9 100644
--- a/packages/backend-core/src/events/publishers/datasource.ts
+++ b/packages/backend-core/src/events/publishers/datasource.ts
@@ -5,8 +5,15 @@ import {
DatasourceCreatedEvent,
DatasourceUpdatedEvent,
DatasourceDeletedEvent,
+ SourceName,
} from "@budibase/types"
+function isCustom(datasource: Datasource) {
+ const sources = Object.values(SourceName)
+ // if not in the base source list, then it must be custom
+ return !sources.includes(datasource.source)
+}
+
export async function created(
datasource: Datasource,
timestamp?: string | number
@@ -14,6 +21,7 @@ export async function created(
const properties: DatasourceCreatedEvent = {
datasourceId: datasource._id as string,
source: datasource.source,
+ custom: isCustom(datasource),
}
await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
}
@@ -22,6 +30,7 @@ export async function updated(datasource: Datasource) {
const properties: DatasourceUpdatedEvent = {
datasourceId: datasource._id as string,
source: datasource.source,
+ custom: isCustom(datasource),
}
await publishEvent(Event.DATASOURCE_UPDATED, properties)
}
@@ -30,6 +39,7 @@ export async function deleted(datasource: Datasource) {
const properties: DatasourceDeletedEvent = {
datasourceId: datasource._id as string,
source: datasource.source,
+ custom: isCustom(datasource),
}
await publishEvent(Event.DATASOURCE_DELETED, properties)
}
diff --git a/packages/backend-core/src/events/publishers/group.ts b/packages/backend-core/src/events/publishers/group.ts
index d300873725..b4fd0d1469 100644
--- a/packages/backend-core/src/events/publishers/group.ts
+++ b/packages/backend-core/src/events/publishers/group.ts
@@ -40,9 +40,9 @@ export async function usersAdded(count: number, group: UserGroup) {
await publishEvent(Event.USER_GROUP_USERS_ADDED, properties)
}
-export async function usersDeleted(emails: string[], group: UserGroup) {
+export async function usersDeleted(count: number, group: UserGroup) {
const properties: GroupUsersDeletedEvent = {
- count: emails.length,
+ count,
groupId: group._id as string,
}
await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties)
diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts
index 57fd0bf8e2..7306312a8f 100644
--- a/packages/backend-core/src/events/publishers/index.ts
+++ b/packages/backend-core/src/events/publishers/index.ts
@@ -18,3 +18,5 @@ export * as view from "./view"
export * as installation from "./installation"
export * as backfill from "./backfill"
export * as group from "./group"
+export * as plugin from "./plugin"
+export * as backup from "./backup"
diff --git a/packages/backend-core/src/events/publishers/license.ts b/packages/backend-core/src/events/publishers/license.ts
index 1adc71652e..84472e408f 100644
--- a/packages/backend-core/src/events/publishers/license.ts
+++ b/packages/backend-core/src/events/publishers/license.ts
@@ -1,27 +1,78 @@
import { publishEvent } from "../events"
import {
Event,
- License,
LicenseActivatedEvent,
- LicenseDowngradedEvent,
- LicenseUpdatedEvent,
- LicenseUpgradedEvent,
+ LicensePlanChangedEvent,
+ LicenseTierChangedEvent,
+ PlanType,
+ Account,
+ LicensePortalOpenedEvent,
+ LicenseCheckoutSuccessEvent,
+ LicenseCheckoutOpenedEvent,
+ LicensePaymentFailedEvent,
+ LicensePaymentRecoveredEvent,
} from "@budibase/types"
-// TODO
-export async function updgraded(license: License) {
- const properties: LicenseUpgradedEvent = {}
- await publishEvent(Event.LICENSE_UPGRADED, properties)
+export async function tierChanged(account: Account, from: number, to: number) {
+ const properties: LicenseTierChangedEvent = {
+ accountId: account.accountId,
+ to,
+ from,
+ }
+ await publishEvent(Event.LICENSE_TIER_CHANGED, properties)
}
-// TODO
-export async function downgraded(license: License) {
- const properties: LicenseDowngradedEvent = {}
- await publishEvent(Event.LICENSE_DOWNGRADED, properties)
+export async function planChanged(
+ account: Account,
+ from: PlanType,
+ to: PlanType
+) {
+ const properties: LicensePlanChangedEvent = {
+ accountId: account.accountId,
+ to,
+ from,
+ }
+ await publishEvent(Event.LICENSE_PLAN_CHANGED, properties)
}
-// TODO
-export async function activated(license: License) {
- const properties: LicenseActivatedEvent = {}
+export async function activated(account: Account) {
+ const properties: LicenseActivatedEvent = {
+ accountId: account.accountId,
+ }
await publishEvent(Event.LICENSE_ACTIVATED, properties)
}
+
+export async function checkoutOpened(account: Account) {
+ const properties: LicenseCheckoutOpenedEvent = {
+ accountId: account.accountId,
+ }
+ await publishEvent(Event.LICENSE_CHECKOUT_OPENED, properties)
+}
+
+export async function checkoutSuccess(account: Account) {
+ const properties: LicenseCheckoutSuccessEvent = {
+ accountId: account.accountId,
+ }
+ await publishEvent(Event.LICENSE_CHECKOUT_SUCCESS, properties)
+}
+
+export async function portalOpened(account: Account) {
+ const properties: LicensePortalOpenedEvent = {
+ accountId: account.accountId,
+ }
+ await publishEvent(Event.LICENSE_PORTAL_OPENED, properties)
+}
+
+export async function paymentFailed(account: Account) {
+ const properties: LicensePaymentFailedEvent = {
+ accountId: account.accountId,
+ }
+ await publishEvent(Event.LICENSE_PAYMENT_FAILED, properties)
+}
+
+export async function paymentRecovered(account: Account) {
+ const properties: LicensePaymentRecoveredEvent = {
+ accountId: account.accountId,
+ }
+ await publishEvent(Event.LICENSE_PAYMENT_RECOVERED, properties)
+}
diff --git a/packages/backend-core/src/events/publishers/plugin.ts b/packages/backend-core/src/events/publishers/plugin.ts
new file mode 100644
index 0000000000..4e4d87cf56
--- /dev/null
+++ b/packages/backend-core/src/events/publishers/plugin.ts
@@ -0,0 +1,41 @@
+import { publishEvent } from "../events"
+import {
+ Event,
+ Plugin,
+ PluginDeletedEvent,
+ PluginImportedEvent,
+ PluginInitEvent,
+} from "@budibase/types"
+
+export async function init(plugin: Plugin) {
+ const properties: PluginInitEvent = {
+ type: plugin.schema.type,
+ name: plugin.name,
+ description: plugin.description,
+ version: plugin.version,
+ }
+ await publishEvent(Event.PLUGIN_INIT, properties)
+}
+
+export async function imported(plugin: Plugin) {
+ const properties: PluginImportedEvent = {
+ pluginId: plugin._id as string,
+ type: plugin.schema.type,
+ source: plugin.source,
+ name: plugin.name,
+ description: plugin.description,
+ version: plugin.version,
+ }
+ await publishEvent(Event.PLUGIN_IMPORTED, properties)
+}
+
+export async function deleted(plugin: Plugin) {
+ const properties: PluginDeletedEvent = {
+ pluginId: plugin._id as string,
+ type: plugin.schema.type,
+ name: plugin.name,
+ description: plugin.description,
+ version: plugin.version,
+ }
+ await publishEvent(Event.PLUGIN_DELETED, properties)
+}
diff --git a/packages/backend-core/src/events/publishers/serve.ts b/packages/backend-core/src/events/publishers/serve.ts
index 13afede029..128e0b9b11 100644
--- a/packages/backend-core/src/events/publishers/serve.ts
+++ b/packages/backend-core/src/events/publishers/serve.ts
@@ -7,22 +7,26 @@ import {
AppServedEvent,
} from "@budibase/types"
-export async function servedBuilder() {
- const properties: BuilderServedEvent = {}
+export async function servedBuilder(timezone: string) {
+ const properties: BuilderServedEvent = {
+ timezone,
+ }
await publishEvent(Event.SERVED_BUILDER, properties)
}
-export async function servedApp(app: App) {
+export async function servedApp(app: App, timezone: string) {
const properties: AppServedEvent = {
appVersion: app.version,
+ timezone,
}
await publishEvent(Event.SERVED_APP, properties)
}
-export async function servedAppPreview(app: App) {
+export async function servedAppPreview(app: App, timezone: string) {
const properties: AppPreviewServedEvent = {
appId: app.appId,
appVersion: app.version,
+ timezone,
}
await publishEvent(Event.SERVED_APP_PREVIEW, properties)
}
diff --git a/packages/backend-core/src/featureFlags/index.js b/packages/backend-core/src/featureFlags/index.js
deleted file mode 100644
index 103ac4df59..0000000000
--- a/packages/backend-core/src/featureFlags/index.js
+++ /dev/null
@@ -1,54 +0,0 @@
-const env = require("../environment")
-const tenancy = require("../tenancy")
-
-/**
- * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
- * The env var is formatted as:
- * tenant1:feature1:feature2,tenant2:feature1
- */
-const getFeatureFlags = () => {
- if (!env.TENANT_FEATURE_FLAGS) {
- return
- }
-
- const tenantFeatureFlags = {}
-
- env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => {
- const [tenantId, ...features] = tenantToFeatures.split(":")
-
- features.forEach(feature => {
- if (!tenantFeatureFlags[tenantId]) {
- tenantFeatureFlags[tenantId] = []
- }
- tenantFeatureFlags[tenantId].push(feature)
- })
- })
-
- return tenantFeatureFlags
-}
-
-const TENANT_FEATURE_FLAGS = getFeatureFlags()
-
-exports.isEnabled = featureFlag => {
- const tenantId = tenancy.getTenantId()
-
- return (
- TENANT_FEATURE_FLAGS &&
- TENANT_FEATURE_FLAGS[tenantId] &&
- TENANT_FEATURE_FLAGS[tenantId].includes(featureFlag)
- )
-}
-
-exports.getTenantFeatureFlags = tenantId => {
- if (TENANT_FEATURE_FLAGS && TENANT_FEATURE_FLAGS[tenantId]) {
- return TENANT_FEATURE_FLAGS[tenantId]
- }
-
- return []
-}
-
-exports.FeatureFlag = {
- LICENSING: "LICENSING",
- GOOGLE_SHEETS: "GOOGLE_SHEETS",
- USER_GROUPS: "USER_GROUPS",
-}
diff --git a/packages/backend-core/src/featureFlags/index.ts b/packages/backend-core/src/featureFlags/index.ts
new file mode 100644
index 0000000000..71e226c976
--- /dev/null
+++ b/packages/backend-core/src/featureFlags/index.ts
@@ -0,0 +1,60 @@
+import env from "../environment"
+import * as tenancy from "../tenancy"
+
+/**
+ * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
+ * The env var is formatted as:
+ * tenant1:feature1:feature2,tenant2:feature1
+ */
+function getFeatureFlags() {
+ if (!env.TENANT_FEATURE_FLAGS) {
+ return
+ }
+
+ const tenantFeatureFlags: Record = {}
+
+ env.TENANT_FEATURE_FLAGS.split(",").forEach(tenantToFeatures => {
+ const [tenantId, ...features] = tenantToFeatures.split(":")
+
+ features.forEach(feature => {
+ if (!tenantFeatureFlags[tenantId]) {
+ tenantFeatureFlags[tenantId] = []
+ }
+ tenantFeatureFlags[tenantId].push(feature)
+ })
+ })
+
+ return tenantFeatureFlags
+}
+
+const TENANT_FEATURE_FLAGS = getFeatureFlags()
+
+export function isEnabled(featureFlag: string) {
+ const tenantId = tenancy.getTenantId()
+ const flags = getTenantFeatureFlags(tenantId)
+ return flags.includes(featureFlag)
+}
+
+export function getTenantFeatureFlags(tenantId: string) {
+ const flags = []
+
+ if (TENANT_FEATURE_FLAGS) {
+ const globalFlags = TENANT_FEATURE_FLAGS["*"]
+ const tenantFlags = TENANT_FEATURE_FLAGS[tenantId]
+
+ if (globalFlags) {
+ flags.push(...globalFlags)
+ }
+ if (tenantFlags) {
+ flags.push(...tenantFlags)
+ }
+ }
+
+ return flags
+}
+
+export enum TenantFeatureFlag {
+ LICENSING = "LICENSING",
+ GOOGLE_SHEETS = "GOOGLE_SHEETS",
+ USER_GROUPS = "USER_GROUPS",
+}
diff --git a/packages/backend-core/src/hashing.js b/packages/backend-core/src/hashing.js
deleted file mode 100644
index 45abe2f9bd..0000000000
--- a/packages/backend-core/src/hashing.js
+++ /dev/null
@@ -1,18 +0,0 @@
-const bcrypt = require("bcrypt")
-const env = require("./environment")
-const { v4 } = require("uuid")
-
-const SALT_ROUNDS = env.SALT_ROUNDS || 10
-
-exports.hash = async data => {
- const salt = await bcrypt.genSalt(SALT_ROUNDS)
- return bcrypt.hash(data, salt)
-}
-
-exports.compare = async (data, encrypted) => {
- return bcrypt.compare(data, encrypted)
-}
-
-exports.newid = function () {
- return v4().replace(/-/g, "")
-}
diff --git a/packages/backend-core/src/helpers.js b/packages/backend-core/src/helpers.ts
similarity index 85%
rename from packages/backend-core/src/helpers.js
rename to packages/backend-core/src/helpers.ts
index b402a82cf3..e1e065bd4e 100644
--- a/packages/backend-core/src/helpers.js
+++ b/packages/backend-core/src/helpers.ts
@@ -4,6 +4,6 @@
* @param {string} url The URL to test and remove any extra double slashes.
* @return {string} The updated url.
*/
-exports.checkSlashesInUrl = url => {
+export function checkSlashesInUrl(url: string) {
return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
}
diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index e585d4b6c3..a4d4ad0a80 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -4,26 +4,28 @@ import * as events from "./events"
import * as migrations from "./migrations"
import * as users from "./users"
import * as roles from "./security/roles"
+import * as permissions from "./security/permissions"
import * as accounts from "./cloud/accounts"
import * as installation from "./installation"
import env from "./environment"
-import tenancy from "./tenancy"
-import featureFlags from "./featureFlags"
+import * as tenancy from "./tenancy"
+import * as featureFlags from "./featureFlags"
import * as sessions from "./security/sessions"
-import deprovisioning from "./context/deprovision"
-import auth from "./auth"
-import constants from "./constants"
-import * as dbConstants from "./db/constants"
-import logging from "./logging"
-import pino from "./pino"
-
-// mimic the outer package exports
-import * as db from "./pkg/db"
-import * as objectStore from "./pkg/objectStore"
-import * as utils from "./pkg/utils"
-import redis from "./pkg/redis"
-import cache from "./pkg/cache"
-import context from "./pkg/context"
+import * as deprovisioning from "./context/deprovision"
+import * as auth from "./auth"
+import * as constants from "./constants"
+import * as logging from "./logging"
+import * as pino from "./pino"
+import * as middleware from "./middleware"
+import * as plugins from "./plugin"
+import * as encryption from "./security/encryption"
+import * as queue from "./queue"
+import * as db from "./db"
+import * as context from "./context"
+import * as cache from "./cache"
+import * as objectStore from "./objectStore"
+import * as redis from "./redis"
+import * as utils from "./utils"
const init = (opts: any = {}) => {
db.init(opts.db)
@@ -32,15 +34,15 @@ const init = (opts: any = {}) => {
const core = {
init,
db,
- ...dbConstants,
+ ...constants,
redis,
+ locks: redis.redlock,
objectStore,
utils,
users,
cache,
auth,
constants,
- ...constants,
migrations,
env,
accounts,
@@ -54,8 +56,13 @@ const core = {
errors,
logging,
roles,
+ plugins,
...pino,
...errorClasses,
+ middleware,
+ encryption,
+ queue,
+ permissions,
}
export = core
diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts
index da9b6c5b76..4e78a508a5 100644
--- a/packages/backend-core/src/installation.ts
+++ b/packages/backend-core/src/installation.ts
@@ -1,16 +1,16 @@
-import * as hashing from "./hashing"
+import { newid } from "./utils"
import * as events from "./events"
-import { StaticDatabases } from "./db/constants"
+import { StaticDatabases } from "./db"
import { doWithDB } from "./db"
import { Installation, IdentityType } from "@budibase/types"
import * as context from "./context"
import semver from "semver"
-import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic"
+import { bustCache, withCache, TTL, CacheKey } from "./cache/generic"
const pkg = require("../package.json")
export const getInstall = async (): Promise => {
- return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
+ return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
useTenancy: false,
})
}
@@ -28,7 +28,7 @@ const getInstallFromDB = async (): Promise => {
if (e.status === 404) {
install = {
_id: StaticDatabases.PLATFORM_INFO.docs.install,
- installId: hashing.newid(),
+ installId: newid(),
version: pkg.version,
}
const resp = await platformDb.put(install)
@@ -50,7 +50,7 @@ const updateVersion = async (version: string): Promise => {
const install = await getInstall()
install.version = version
await platformDb.put(install)
- await bustCache(CacheKeys.INSTALLATION)
+ await bustCache(CacheKey.INSTALLATION)
}
)
} catch (e: any) {
diff --git a/packages/backend-core/src/middleware/adminOnly.js b/packages/backend-core/src/middleware/adminOnly.ts
similarity index 63%
rename from packages/backend-core/src/middleware/adminOnly.js
rename to packages/backend-core/src/middleware/adminOnly.ts
index 4bfdf83848..30fdf2907b 100644
--- a/packages/backend-core/src/middleware/adminOnly.js
+++ b/packages/backend-core/src/middleware/adminOnly.ts
@@ -1,4 +1,6 @@
-module.exports = async (ctx, next) => {
+import { BBContext } from "@budibase/types"
+
+export = async (ctx: BBContext, next: any) => {
if (
!ctx.internal &&
(!ctx.user || !ctx.user.admin || !ctx.user.admin.global)
diff --git a/packages/backend-core/src/middleware/auditLog.js b/packages/backend-core/src/middleware/auditLog.js
deleted file mode 100644
index c9063ae2e0..0000000000
--- a/packages/backend-core/src/middleware/auditLog.js
+++ /dev/null
@@ -1,4 +0,0 @@
-module.exports = async (ctx, next) => {
- // Placeholder for audit log middleware
- return next()
-}
diff --git a/packages/backend-core/src/middleware/auditLog.ts b/packages/backend-core/src/middleware/auditLog.ts
new file mode 100644
index 0000000000..a2c30ade8a
--- /dev/null
+++ b/packages/backend-core/src/middleware/auditLog.ts
@@ -0,0 +1,6 @@
+import { BBContext } from "@budibase/types"
+
+export = async (ctx: BBContext | any, next: any) => {
+ // Placeholder for audit log middleware
+ return next()
+}
diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts
index 3406b00812..fcf07c50a5 100644
--- a/packages/backend-core/src/middleware/authenticated.ts
+++ b/packages/backend-core/src/middleware/authenticated.ts
@@ -1,17 +1,18 @@
-import { Cookies, Headers } from "../constants"
+import { Cookie, Header } from "../constants"
import { getCookie, clearCookie, openJwt } from "../utils"
import { getUser } from "../cache/user"
import { getSession, updateSessionTTL } from "../security/sessions"
import { buildMatcherRegex, matches } from "./matchers"
-import { SEPARATOR } from "../db/constants"
-import { ViewNames } from "../db/utils"
-import { queryGlobalView } from "../db/views"
+import { SEPARATOR, queryGlobalView, ViewName } from "../db"
import { getGlobalDB, doInTenant } from "../tenancy"
import { decrypt } from "../security/encryption"
-const identity = require("../context/identity")
-const env = require("../environment")
+import * as identity from "../context/identity"
+import env from "../environment"
+import { BBContext, EndpointMatcher } from "@budibase/types"
-const ONE_MINUTE = env.SESSION_UPDATE_PERIOD || 60 * 1000
+const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
+ ? parseInt(env.SESSION_UPDATE_PERIOD)
+ : 60 * 1000
interface FinaliseOpts {
authenticated?: boolean
@@ -42,13 +43,13 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
return doInTenant(tenantId, async () => {
const db = getGlobalDB()
// api key is encrypted in the database
- const userId = await queryGlobalView(
- ViewNames.BY_API_KEY,
+ const userId = (await queryGlobalView(
+ ViewName.BY_API_KEY,
{
key: apiKey,
},
db
- )
+ )) as string
if (userId) {
return {
valid: true,
@@ -65,16 +66,16 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
* The tenancy modules should not be used here and it should be assumed that the tenancy context
* has not yet been populated.
*/
-module.exports = (
- noAuthPatterns = [],
- opts: { publicAllowed: boolean; populateUser?: Function } = {
+export = function (
+ noAuthPatterns: EndpointMatcher[] = [],
+ opts: { publicAllowed?: boolean; populateUser?: Function } = {
publicAllowed: false,
}
-) => {
+) {
const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
- return async (ctx: any, next: any) => {
+ return async (ctx: BBContext | any, next: any) => {
let publicEndpoint = false
- const version = ctx.request.headers[Headers.API_VER]
+ const version = ctx.request.headers[Header.API_VER]
// the path is not authenticated
const found = matches(ctx, noAuthOptions)
if (found) {
@@ -82,10 +83,10 @@ module.exports = (
}
try {
// check the actual user is authenticated first, try header or cookie
- const headerToken = ctx.request.headers[Headers.TOKEN]
- const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
- const apiKey = ctx.request.headers[Headers.API_KEY]
- const tenantId = ctx.request.headers[Headers.TENANT_ID]
+ const headerToken = ctx.request.headers[Header.TOKEN]
+ const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
+ const apiKey = ctx.request.headers[Header.API_KEY]
+ const tenantId = ctx.request.headers[Header.TENANT_ID]
let authenticated = false,
user = null,
internal = false
@@ -106,6 +107,7 @@ module.exports = (
user = await getUser(userId, session.tenantId)
}
user.csrfToken = session.csrfToken
+
if (session?.lastAccessedAt < timeMinusOneMinute()) {
// make sure we denote that the session is still in use
await updateSessionTTL(session)
@@ -115,7 +117,7 @@ module.exports = (
authenticated = false
console.error("Auth Error", err?.message || err)
// remove the cookie as the user does not exist anymore
- clearCookie(ctx, Cookies.Auth)
+ clearCookie(ctx, Cookie.Auth)
}
}
// this is an internal request, no user made it
@@ -139,7 +141,7 @@ module.exports = (
delete user.password
}
// be explicit
- if (authenticated !== true) {
+ if (!authenticated) {
authenticated = false
}
// isAuthenticated is a function, so use a variable to be able to check authed state
@@ -151,9 +153,10 @@ module.exports = (
return next()
}
} catch (err: any) {
+ console.error("Auth Error", err?.message || err)
// invalid token, clear the cookie
if (err && err.name === "JsonWebTokenError") {
- clearCookie(ctx, Cookies.Auth)
+ clearCookie(ctx, Cookie.Auth)
}
// allow configuring for public access
if ((opts && opts.publicAllowed) || publicEndpoint) {
diff --git a/packages/backend-core/src/middleware/builderOnly.js b/packages/backend-core/src/middleware/builderOnly.ts
similarity index 64%
rename from packages/backend-core/src/middleware/builderOnly.js
rename to packages/backend-core/src/middleware/builderOnly.ts
index 2128626db4..e13882d7f6 100644
--- a/packages/backend-core/src/middleware/builderOnly.js
+++ b/packages/backend-core/src/middleware/builderOnly.ts
@@ -1,4 +1,6 @@
-module.exports = async (ctx, next) => {
+import { BBContext } from "@budibase/types"
+
+export = async (ctx: BBContext, next: any) => {
if (
!ctx.internal &&
(!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
diff --git a/packages/backend-core/src/middleware/builderOrAdmin.js b/packages/backend-core/src/middleware/builderOrAdmin.ts
similarity index 71%
rename from packages/backend-core/src/middleware/builderOrAdmin.js
rename to packages/backend-core/src/middleware/builderOrAdmin.ts
index 6440766298..26664695f8 100644
--- a/packages/backend-core/src/middleware/builderOrAdmin.js
+++ b/packages/backend-core/src/middleware/builderOrAdmin.ts
@@ -1,4 +1,6 @@
-module.exports = async (ctx, next) => {
+import { BBContext } from "@budibase/types"
+
+export = async (ctx: BBContext, next: any) => {
if (
!ctx.internal &&
(!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&
diff --git a/packages/backend-core/src/middleware/csrf.js b/packages/backend-core/src/middleware/csrf.ts
similarity index 84%
rename from packages/backend-core/src/middleware/csrf.js
rename to packages/backend-core/src/middleware/csrf.ts
index 12bd9473e6..654ba47e07 100644
--- a/packages/backend-core/src/middleware/csrf.js
+++ b/packages/backend-core/src/middleware/csrf.ts
@@ -1,5 +1,6 @@
-const { Headers } = require("../constants")
-const { buildMatcherRegex, matches } = require("./matchers")
+import { Header } from "../constants"
+import { buildMatcherRegex, matches } from "./matchers"
+import { BBContext, EndpointMatcher } from "@budibase/types"
/**
* GET, HEAD and OPTIONS methods are considered safe operations
@@ -31,9 +32,11 @@ const INCLUDED_CONTENT_TYPES = [
* https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#synchronizer-token-pattern
*
*/
-module.exports = (opts = { noCsrfPatterns: [] }) => {
+export = function (
+ opts: { noCsrfPatterns: EndpointMatcher[] } = { noCsrfPatterns: [] }
+) {
const noCsrfOptions = buildMatcherRegex(opts.noCsrfPatterns)
- return async (ctx, next) => {
+ return async (ctx: BBContext | any, next: any) => {
// don't apply for excluded paths
const found = matches(ctx, noCsrfOptions)
if (found) {
@@ -62,13 +65,13 @@ module.exports = (opts = { noCsrfPatterns: [] }) => {
// apply csrf when there is a token in the session (new logins)
// in future there should be a hard requirement that the token is present
- const userToken = ctx.user.csrfToken
+ const userToken = ctx.user?.csrfToken
if (!userToken) {
return next()
}
// reject if no token in request or mismatch
- const requestToken = ctx.get(Headers.CSRF_TOKEN)
+ const requestToken = ctx.get(Header.CSRF_TOKEN)
if (!requestToken || requestToken !== userToken) {
ctx.throw(403, "Invalid CSRF token")
}
diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js
deleted file mode 100644
index 7e7b8a2931..0000000000
--- a/packages/backend-core/src/middleware/index.js
+++ /dev/null
@@ -1,35 +0,0 @@
-const jwt = require("./passport/jwt")
-const local = require("./passport/local")
-const google = require("./passport/google")
-const oidc = require("./passport/oidc")
-const { authError, ssoCallbackUrl } = require("./passport/utils")
-const authenticated = require("./authenticated")
-const auditLog = require("./auditLog")
-const tenancy = require("./tenancy")
-const internalApi = require("./internalApi")
-const datasourceGoogle = require("./passport/datasource/google")
-const csrf = require("./csrf")
-const adminOnly = require("./adminOnly")
-const builderOrAdmin = require("./builderOrAdmin")
-const builderOnly = require("./builderOnly")
-const joiValidator = require("./joi-validator")
-module.exports = {
- google,
- oidc,
- jwt,
- local,
- authenticated,
- auditLog,
- tenancy,
- authError,
- internalApi,
- ssoCallbackUrl,
- datasource: {
- google: datasourceGoogle,
- },
- csrf,
- adminOnly,
- builderOnly,
- builderOrAdmin,
- joiValidator,
-}
diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts
new file mode 100644
index 0000000000..2b332f5c49
--- /dev/null
+++ b/packages/backend-core/src/middleware/index.ts
@@ -0,0 +1,38 @@
+import * as jwt from "./passport/jwt"
+import * as local from "./passport/local"
+import * as google from "./passport/google"
+import * as oidc from "./passport/oidc"
+import { authError, ssoCallbackUrl } from "./passport/utils"
+import authenticated from "./authenticated"
+import auditLog from "./auditLog"
+import tenancy from "./tenancy"
+import internalApi from "./internalApi"
+import * as datasourceGoogle from "./passport/datasource/google"
+import csrf from "./csrf"
+import adminOnly from "./adminOnly"
+import builderOrAdmin from "./builderOrAdmin"
+import builderOnly from "./builderOnly"
+import * as joiValidator from "./joi-validator"
+
+const pkg = {
+ google,
+ oidc,
+ jwt,
+ local,
+ authenticated,
+ auditLog,
+ tenancy,
+ authError,
+ internalApi,
+ ssoCallbackUrl,
+ datasource: {
+ google: datasourceGoogle,
+ },
+ csrf,
+ adminOnly,
+ builderOnly,
+ builderOrAdmin,
+ joiValidator,
+}
+
+export = pkg
diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js
deleted file mode 100644
index 275d559a9e..0000000000
--- a/packages/backend-core/src/middleware/internalApi.js
+++ /dev/null
@@ -1,14 +0,0 @@
-const env = require("../environment")
-const { Headers } = require("../constants")
-
-/**
- * API Key only endpoint.
- */
-module.exports = async (ctx, next) => {
- const apiKey = ctx.request.headers[Headers.API_KEY]
- if (apiKey !== env.INTERNAL_API_KEY) {
- ctx.throw(403, "Unauthorized")
- }
-
- return next()
-}
diff --git a/packages/backend-core/src/middleware/internalApi.ts b/packages/backend-core/src/middleware/internalApi.ts
new file mode 100644
index 0000000000..f4f08ec2dd
--- /dev/null
+++ b/packages/backend-core/src/middleware/internalApi.ts
@@ -0,0 +1,15 @@
+import env from "../environment"
+import { Header } from "../constants"
+import { BBContext } from "@budibase/types"
+
+/**
+ * API Key only endpoint.
+ */
+export = async (ctx: BBContext, next: any) => {
+ const apiKey = ctx.request.headers[Header.API_KEY]
+ if (apiKey !== env.INTERNAL_API_KEY) {
+ ctx.throw(403, "Unauthorized")
+ }
+
+ return next()
+}
diff --git a/packages/backend-core/src/middleware/joi-validator.js b/packages/backend-core/src/middleware/joi-validator.js
deleted file mode 100644
index 1686b0e727..0000000000
--- a/packages/backend-core/src/middleware/joi-validator.js
+++ /dev/null
@@ -1,28 +0,0 @@
-function validate(schema, property) {
- // Return a Koa middleware function
- return (ctx, next) => {
- if (!schema) {
- return next()
- }
- let params = null
- if (ctx[property] != null) {
- params = ctx[property]
- } else if (ctx.request[property] != null) {
- params = ctx.request[property]
- }
- const { error } = schema.validate(params)
- if (error) {
- ctx.throw(400, `Invalid ${property} - ${error.message}`)
- return
- }
- return next()
- }
-}
-
-module.exports.body = schema => {
- return validate(schema, "body")
-}
-
-module.exports.params = schema => {
- return validate(schema, "params")
-}
diff --git a/packages/backend-core/src/middleware/joi-validator.ts b/packages/backend-core/src/middleware/joi-validator.ts
new file mode 100644
index 0000000000..fcc8316886
--- /dev/null
+++ b/packages/backend-core/src/middleware/joi-validator.ts
@@ -0,0 +1,45 @@
+import Joi, { ObjectSchema } from "joi"
+import { BBContext } from "@budibase/types"
+
+function validate(
+ schema: Joi.ObjectSchema | Joi.ArraySchema,
+ property: string
+) {
+ // Return a Koa middleware function
+ return (ctx: BBContext, next: any) => {
+ if (!schema) {
+ return next()
+ }
+ let params = null
+ // @ts-ignore
+ let reqProp = ctx.request?.[property]
+ if (ctx[property] != null) {
+ params = ctx[property]
+ } else if (reqProp != null) {
+ params = reqProp
+ }
+
+ // not all schemas have the append property e.g. array schemas
+ if ((schema as Joi.ObjectSchema).append) {
+ schema = (schema as Joi.ObjectSchema).append({
+ createdAt: Joi.any().optional(),
+ updatedAt: Joi.any().optional(),
+ })
+ }
+
+ const { error } = schema.validate(params)
+ if (error) {
+ ctx.throw(400, `Invalid ${property} - ${error.message}`)
+ return
+ }
+ return next()
+ }
+}
+
+export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+ return validate(schema, "body")
+}
+
+export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+ return validate(schema, "params")
+}
diff --git a/packages/backend-core/src/middleware/matchers.js b/packages/backend-core/src/middleware/matchers.ts
similarity index 62%
rename from packages/backend-core/src/middleware/matchers.js
rename to packages/backend-core/src/middleware/matchers.ts
index 3d5065c069..efbdec2dbe 100644
--- a/packages/backend-core/src/middleware/matchers.js
+++ b/packages/backend-core/src/middleware/matchers.ts
@@ -1,27 +1,34 @@
+import { BBContext, EndpointMatcher, RegexMatcher } from "@budibase/types"
+
const PARAM_REGEX = /\/:(.*?)(\/.*)?$/g
-exports.buildMatcherRegex = patterns => {
+export const buildMatcherRegex = (
+ patterns: EndpointMatcher[]
+): RegexMatcher[] => {
if (!patterns) {
return []
}
return patterns.map(pattern => {
- const isObj = typeof pattern === "object" && pattern.route
- const method = isObj ? pattern.method : "GET"
+ let route = pattern.route
+ const method = pattern.method
const strict = pattern.strict ? pattern.strict : false
- let route = isObj ? pattern.route : pattern
+ // if there is a param in the route
+ // use a wildcard pattern
const matches = route.match(PARAM_REGEX)
if (matches) {
for (let match of matches) {
- const pattern = "/.*" + (match.endsWith("/") ? "/" : "")
+ const suffix = match.endsWith("/") ? "/" : ""
+ const pattern = "/.*" + suffix
route = route.replace(match, pattern)
}
}
+
return { regex: new RegExp(route), method, strict, route }
})
}
-exports.matches = (ctx, options) => {
+export const matches = (ctx: BBContext, options: RegexMatcher[]) => {
return options.find(({ regex, method, strict, route }) => {
let urlMatch
if (strict) {
diff --git a/packages/backend-core/src/middleware/passport/datasource/google.js b/packages/backend-core/src/middleware/passport/datasource/google.ts
similarity index 67%
rename from packages/backend-core/src/middleware/passport/datasource/google.js
rename to packages/backend-core/src/middleware/passport/datasource/google.ts
index 8f2022c2d7..65620d7aa3 100644
--- a/packages/backend-core/src/middleware/passport/datasource/google.js
+++ b/packages/backend-core/src/middleware/passport/datasource/google.ts
@@ -1,17 +1,21 @@
-const google = require("../google")
+import * as google from "../google"
+import { Cookie, Config } from "../../../constants"
+import { clearCookie, getCookie } from "../../../utils"
+import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
+import environment from "../../../environment"
+import { getGlobalDB } from "../../../tenancy"
+import { BBContext, Database, SSOProfile } from "@budibase/types"
const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
-const { Cookies, Configs } = require("../../../constants")
-const { clearCookie, getCookie } = require("../../../utils")
-const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
-const { doWithDB } = require("../../../db")
-const environment = require("../../../environment")
-const { getGlobalDB } = require("../../../tenancy")
+
+type Passport = {
+ authenticate: any
+}
async function fetchGoogleCreds() {
// try and get the config from the tenant
const db = getGlobalDB()
const googleConfig = await getScopedConfig(db, {
- type: Configs.GOOGLE,
+ type: Config.GOOGLE,
})
// or fall back to env variables
return (
@@ -22,7 +26,11 @@ async function fetchGoogleCreds() {
)
}
-async function preAuth(passport, ctx, next) {
+export async function preAuth(
+ passport: Passport,
+ ctx: BBContext,
+ next: Function
+) {
// get the relevant config
const googleConfig = await fetchGoogleCreds()
const platformUrl = await getPlatformUrl({ tenantAware: false })
@@ -41,13 +49,17 @@ async function preAuth(passport, ctx, next) {
})(ctx, next)
}
-async function postAuth(passport, ctx, next) {
+export async function postAuth(
+ passport: Passport,
+ ctx: BBContext,
+ next: Function
+) {
// get the relevant config
const config = await fetchGoogleCreds()
const platformUrl = await getPlatformUrl({ tenantAware: false })
let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
- const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
+ const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
return passport.authenticate(
new GoogleStrategy(
@@ -56,15 +68,20 @@ async function postAuth(passport, ctx, next) {
clientSecret: config.clientSecret,
callbackURL: callbackUrl,
},
- (accessToken, refreshToken, profile, done) => {
- clearCookie(ctx, Cookies.DatasourceAuth)
+ (
+ accessToken: string,
+ refreshToken: string,
+ profile: SSOProfile,
+ done: Function
+ ) => {
+ clearCookie(ctx, Cookie.DatasourceAuth)
done(null, { accessToken, refreshToken })
}
),
{ successRedirect: "/", failureRedirect: "/error" },
- async (err, tokens) => {
+ async (err: any, tokens: string[]) => {
// update the DB for the datasource with all the user info
- await doWithDB(authStateCookie.appId, async db => {
+ await doWithDB(authStateCookie.appId, async (db: Database) => {
const datasource = await db.get(authStateCookie.datasourceId)
if (!datasource.config) {
datasource.config = {}
@@ -78,6 +95,3 @@ async function postAuth(passport, ctx, next) {
}
)(ctx, next)
}
-
-exports.preAuth = preAuth
-exports.postAuth = postAuth
diff --git a/packages/backend-core/src/middleware/passport/google.js b/packages/backend-core/src/middleware/passport/google.ts
similarity index 63%
rename from packages/backend-core/src/middleware/passport/google.js
rename to packages/backend-core/src/middleware/passport/google.ts
index 7419974cd7..deba849233 100644
--- a/packages/backend-core/src/middleware/passport/google.js
+++ b/packages/backend-core/src/middleware/passport/google.ts
@@ -1,10 +1,15 @@
+import { ssoCallbackUrl } from "./utils"
+import { authenticateThirdParty } from "./third-party-common"
+import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types"
const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
-const { ssoCallbackUrl } = require("./utils")
-const { authenticateThirdParty } = require("./third-party-common")
-const { Configs } = require("../../../constants")
-const buildVerifyFn = saveUserFn => {
- return (accessToken, refreshToken, profile, done) => {
+export function buildVerifyFn(saveUserFn?: Function) {
+ return (
+ accessToken: string,
+ refreshToken: string,
+ profile: SSOProfile,
+ done: Function
+ ) => {
const thirdPartyUser = {
provider: profile.provider, // should always be 'google'
providerType: "google",
@@ -31,7 +36,11 @@ const buildVerifyFn = saveUserFn => {
* from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.
* @returns Dynamically configured Passport Google Strategy
*/
-exports.strategyFactory = async function (config, callbackUrl, saveUserFn) {
+export async function strategyFactory(
+ config: GoogleConfig["config"],
+ callbackUrl: string,
+ saveUserFn?: Function
+) {
try {
const { clientID, clientSecret } = config
@@ -50,18 +59,15 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) {
},
verify
)
- } catch (err) {
+ } catch (err: any) {
console.error(err)
- throw new Error(
- `Error constructing google authentication strategy: ${err}`,
- err
- )
+ throw new Error(`Error constructing google authentication strategy: ${err}`)
}
}
-exports.getCallbackUrl = async function (db, config) {
- return ssoCallbackUrl(db, config, Configs.GOOGLE)
+export async function getCallbackUrl(
+ db: Database,
+ config: { callbackURL?: string }
+) {
+ return ssoCallbackUrl(db, config, ConfigType.GOOGLE)
}
-
-// expose for testing
-exports.buildVerifyFn = buildVerifyFn
diff --git a/packages/backend-core/src/middleware/passport/jwt.js b/packages/backend-core/src/middleware/passport/jwt.js
deleted file mode 100644
index 690c2ac8a1..0000000000
--- a/packages/backend-core/src/middleware/passport/jwt.js
+++ /dev/null
@@ -1,18 +0,0 @@
-const { Cookies } = require("../../constants")
-const env = require("../../environment")
-const { authError } = require("./utils")
-
-exports.options = {
- secretOrKey: env.JWT_SECRET,
- jwtFromRequest: function (ctx) {
- return ctx.cookies.get(Cookies.Auth)
- },
-}
-
-exports.authenticate = async function (jwt, done) {
- try {
- return done(null, jwt)
- } catch (err) {
- return authError(done, "JWT invalid", err)
- }
-}
diff --git a/packages/backend-core/src/middleware/passport/jwt.ts b/packages/backend-core/src/middleware/passport/jwt.ts
new file mode 100644
index 0000000000..95dc8f2656
--- /dev/null
+++ b/packages/backend-core/src/middleware/passport/jwt.ts
@@ -0,0 +1,19 @@
+import { Cookie } from "../../constants"
+import env from "../../environment"
+import { authError } from "./utils"
+import { BBContext } from "@budibase/types"
+
+export const options = {
+ secretOrKey: env.JWT_SECRET,
+ jwtFromRequest: function (ctx: BBContext) {
+ return ctx.cookies.get(Cookie.Auth)
+ },
+}
+
+export async function authenticate(jwt: Function, done: Function) {
+ try {
+ return done(null, jwt)
+ } catch (err) {
+ return authError(done, "JWT invalid", err)
+ }
+}
diff --git a/packages/backend-core/src/middleware/passport/local.js b/packages/backend-core/src/middleware/passport/local.ts
similarity index 73%
rename from packages/backend-core/src/middleware/passport/local.js
rename to packages/backend-core/src/middleware/passport/local.ts
index b955d29102..8b85d3734c 100644
--- a/packages/backend-core/src/middleware/passport/local.js
+++ b/packages/backend-core/src/middleware/passport/local.ts
@@ -1,18 +1,18 @@
+import { UserStatus } from "../../constants"
+import { compare, newid } from "../../utils"
+import env from "../../environment"
+import * as users from "../../users"
+import { authError } from "./utils"
+import { createASession } from "../../security/sessions"
+import { getTenantId } from "../../tenancy"
+import { BBContext } from "@budibase/types"
const jwt = require("jsonwebtoken")
-const { UserStatus } = require("../../constants")
-const { compare } = require("../../hashing")
-const env = require("../../environment")
-const users = require("../../users")
-const { authError } = require("./utils")
-const { newid } = require("../../hashing")
-const { createASession } = require("../../security/sessions")
-const { getTenantId } = require("../../tenancy")
const INVALID_ERR = "Invalid credentials"
const SSO_NO_PASSWORD = "SSO user does not have a password set"
const EXPIRED = "This account has expired. Please reset your password"
-exports.options = {
+export const options = {
passReqToCallback: true,
}
@@ -24,7 +24,12 @@ exports.options = {
* @param {*} done callback from passport to return user information and errors
* @returns The authenticated user, or errors if they occur
*/
-exports.authenticate = async function (ctx, email, password, done) {
+export async function authenticate(
+ ctx: BBContext,
+ email: string,
+ password: string,
+ done: Function
+) {
if (!email) return authError(done, "Email Required")
if (!password) return authError(done, "Password Required")
@@ -56,9 +61,9 @@ exports.authenticate = async function (ctx, email, password, done) {
const sessionId = newid()
const tenantId = getTenantId()
- await createASession(dbUser._id, { sessionId, tenantId })
+ await createASession(dbUser._id!, { sessionId, tenantId })
- dbUser.token = jwt.sign(
+ const token = jwt.sign(
{
userId: dbUser._id,
sessionId,
@@ -69,7 +74,10 @@ exports.authenticate = async function (ctx, email, password, done) {
// Remove users password in payload
delete dbUser.password
- return done(null, dbUser)
+ return done(null, {
+ ...dbUser,
+ token,
+ })
} else {
return authError(done, INVALID_ERR)
}
diff --git a/packages/backend-core/src/middleware/passport/oidc.js b/packages/backend-core/src/middleware/passport/oidc.ts
similarity index 72%
rename from packages/backend-core/src/middleware/passport/oidc.js
rename to packages/backend-core/src/middleware/passport/oidc.ts
index 20dbd4669b..27c3c647b7 100644
--- a/packages/backend-core/src/middleware/passport/oidc.js
+++ b/packages/backend-core/src/middleware/passport/oidc.ts
@@ -1,10 +1,23 @@
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
+import { authenticateThirdParty } from "./third-party-common"
+import { ssoCallbackUrl } from "./utils"
+import {
+ Config,
+ ConfigType,
+ OIDCInnerCfg,
+ Database,
+ SSOProfile,
+ ThirdPartyUser,
+ OIDCConfiguration,
+} from "@budibase/types"
const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
-const { authenticateThirdParty } = require("./third-party-common")
-const { ssoCallbackUrl } = require("./utils")
-const { Configs } = require("../../../constants")
-const buildVerifyFn = saveUserFn => {
+type JwtClaims = {
+ preferred_username: string
+ email: string
+}
+
+export function buildVerifyFn(saveUserFn?: Function) {
/**
* @param {*} issuer The identity provider base URL
* @param {*} sub The user ID
@@ -17,17 +30,17 @@ const buildVerifyFn = saveUserFn => {
* @param {*} done The passport callback: err, user, info
*/
return async (
- issuer,
- sub,
- profile,
- jwtClaims,
- accessToken,
- refreshToken,
- idToken,
- params,
- done
+ issuer: string,
+ sub: string,
+ profile: SSOProfile,
+ jwtClaims: JwtClaims,
+ accessToken: string,
+ refreshToken: string,
+ idToken: string,
+ params: any,
+ done: Function
) => {
- const thirdPartyUser = {
+ const thirdPartyUser: ThirdPartyUser = {
// store the issuer info to enable sync in future
provider: issuer,
providerType: "oidc",
@@ -53,7 +66,7 @@ const buildVerifyFn = saveUserFn => {
* @param {*} profile The structured profile created by passport using the user info endpoint
* @param {*} jwtClaims The claims returned in the id token
*/
-function getEmail(profile, jwtClaims) {
+function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) {
// profile not guaranteed to contain email e.g. github connected azure ad account
if (profile._json.email) {
return profile._json.email
@@ -77,7 +90,7 @@ function getEmail(profile, jwtClaims) {
)
}
-function validEmail(value) {
+function validEmail(value: string) {
return (
value &&
!!value.match(
@@ -91,19 +104,25 @@ function validEmail(value) {
* from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport.
* @returns Dynamically configured Passport OIDC Strategy
*/
-exports.strategyFactory = async function (config, saveUserFn) {
+export async function strategyFactory(
+ config: OIDCConfiguration,
+ saveUserFn?: Function
+) {
try {
const verify = buildVerifyFn(saveUserFn)
const strategy = new OIDCStrategy(config, verify)
strategy.name = "oidc"
return strategy
- } catch (err) {
+ } catch (err: any) {
console.error(err)
- throw new Error("Error constructing OIDC authentication strategy", err)
+ throw new Error(`Error constructing OIDC authentication strategy - ${err}`)
}
}
-exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) {
+export async function fetchStrategyConfig(
+ enrichedConfig: OIDCInnerCfg,
+ callbackUrl?: string
+): Promise {
try {
const { clientID, clientSecret, configUrl } = enrichedConfig
@@ -135,13 +154,15 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) {
}
} catch (err) {
console.error(err)
- throw new Error("Error constructing OIDC authentication configuration", err)
+ throw new Error(
+ `Error constructing OIDC authentication configuration - ${err}`
+ )
}
}
-exports.getCallbackUrl = async function (db, config) {
- return ssoCallbackUrl(db, config, Configs.OIDC)
+export async function getCallbackUrl(
+ db: Database,
+ config: { callbackURL?: string }
+) {
+ return ssoCallbackUrl(db, config, ConfigType.OIDC)
}
-
-// expose for testing
-exports.buildVerifyFn = buildVerifyFn
diff --git a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js
index c00ab2ea7d..4c8aa94ddf 100644
--- a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js
+++ b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js
@@ -1,7 +1,6 @@
// Mock data
-
+const mockFetch = require("node-fetch")
const { data } = require("./utilities/mock-data")
-
const issuer = "mockIssuer"
const sub = "mockSub"
const profile = {
@@ -39,8 +38,6 @@ describe("oidc", () => {
const mockStrategy = require("@techpass/passport-openidconnect").Strategy
// mock the request to retrieve the oidc configuration
- jest.mock("node-fetch")
- const mockFetch = require("node-fetch")
mockFetch.mockReturnValue({
ok: true,
json: () => oidcConfigUrlResponse
diff --git a/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js b/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
index 41a253b328..d377d602f1 100644
--- a/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
+++ b/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
@@ -1,10 +1,10 @@
-require("../../../../tests/utilities/TestConfiguration")
+require("../../../../tests")
const { authenticateThirdParty } = require("../third-party-common")
const { data } = require("./utilities/mock-data")
const { DEFAULT_TENANT_ID } = require("../../../constants")
const { generateGlobalUserID } = require("../../../db/utils")
-const { newid } = require("../../../hashing")
+const { newid } = require("../../../utils")
const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
const done = jest.fn()
diff --git a/packages/backend-core/src/middleware/passport/third-party-common.js b/packages/backend-core/src/middleware/passport/third-party-common.ts
similarity index 77%
rename from packages/backend-core/src/middleware/passport/third-party-common.js
rename to packages/backend-core/src/middleware/passport/third-party-common.ts
index 1c5891fce7..8798ce5298 100644
--- a/packages/backend-core/src/middleware/passport/third-party-common.js
+++ b/packages/backend-core/src/middleware/passport/third-party-common.ts
@@ -1,21 +1,22 @@
-const env = require("../../environment")
+import env from "../../environment"
+import { generateGlobalUserID } from "../../db"
+import { authError } from "./utils"
+import { newid } from "../../utils"
+import { createASession } from "../../security/sessions"
+import * as users from "../../users"
+import { getGlobalDB, getTenantId } from "../../tenancy"
+import fetch from "node-fetch"
+import { ThirdPartyUser } from "@budibase/types"
const jwt = require("jsonwebtoken")
-const { generateGlobalUserID } = require("../../db/utils")
-const { authError } = require("./utils")
-const { newid } = require("../../hashing")
-const { createASession } = require("../../security/sessions")
-const users = require("../../users")
-const { getGlobalDB, getTenantId } = require("../../tenancy")
-const fetch = require("node-fetch")
/**
* Common authentication logic for third parties. e.g. OAuth, OIDC.
*/
-exports.authenticateThirdParty = async function (
- thirdPartyUser,
- requireLocalAccount = true,
- done,
- saveUserFn
+export async function authenticateThirdParty(
+ thirdPartyUser: ThirdPartyUser,
+ requireLocalAccount: boolean = true,
+ done: Function,
+ saveUserFn?: Function
) {
if (!saveUserFn) {
throw new Error("Save user function must be provided")
@@ -39,7 +40,7 @@ exports.authenticateThirdParty = async function (
// try to load by id
try {
dbUser = await db.get(userId)
- } catch (err) {
+ } catch (err: any) {
// abort when not 404 error
if (!err.status || err.status !== 404) {
return authError(
@@ -81,7 +82,7 @@ exports.authenticateThirdParty = async function (
// create or sync the user
try {
await saveUserFn(dbUser, false, false)
- } catch (err) {
+ } catch (err: any) {
return authError(done, err)
}
@@ -104,13 +105,16 @@ exports.authenticateThirdParty = async function (
return done(null, dbUser)
}
-async function syncProfilePicture(user, thirdPartyUser) {
- const pictureUrl = thirdPartyUser.profile._json.picture
+async function syncProfilePicture(
+ user: ThirdPartyUser,
+ thirdPartyUser: ThirdPartyUser
+) {
+ const pictureUrl = thirdPartyUser.profile?._json.picture
if (pictureUrl) {
const response = await fetch(pictureUrl)
if (response.status === 200) {
- const type = response.headers.get("content-type")
+ const type = response.headers.get("content-type") as string
if (type.startsWith("image/")) {
user.pictureUrl = pictureUrl
}
@@ -123,7 +127,7 @@ async function syncProfilePicture(user, thirdPartyUser) {
/**
* @returns a user that has been sync'd with third party information
*/
-async function syncUser(user, thirdPartyUser) {
+async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) {
// provider
user.provider = thirdPartyUser.provider
user.providerType = thirdPartyUser.providerType
diff --git a/packages/backend-core/src/middleware/passport/utils.js b/packages/backend-core/src/middleware/passport/utils.ts
similarity index 64%
rename from packages/backend-core/src/middleware/passport/utils.js
rename to packages/backend-core/src/middleware/passport/utils.ts
index 217130cd6d..3d79aada28 100644
--- a/packages/backend-core/src/middleware/passport/utils.js
+++ b/packages/backend-core/src/middleware/passport/utils.ts
@@ -1,6 +1,6 @@
-const { isMultiTenant, getTenantId } = require("../../tenancy")
-const { getScopedConfig } = require("../../db/utils")
-const { Configs } = require("../../constants")
+import { isMultiTenant, getTenantId } from "../../tenancy"
+import { getScopedConfig } from "../../db"
+import { ConfigType, Database, Config } from "@budibase/types"
/**
* Utility to handle authentication errors.
@@ -10,7 +10,7 @@ const { Configs } = require("../../constants")
* @param {*} err (Optional) error that will be logged
*/
-exports.authError = function (done, message, err = null) {
+export function authError(done: Function, message: string, err?: any) {
return done(
err,
null, // never return a user
@@ -18,13 +18,17 @@ exports.authError = function (done, message, err = null) {
)
}
-exports.ssoCallbackUrl = async (db, config, type) => {
+export async function ssoCallbackUrl(
+ db: Database,
+ config?: { callbackURL?: string },
+ type?: ConfigType
+) {
// incase there is a callback URL from before
if (config && config.callbackURL) {
return config.callbackURL
}
const publicConfig = await getScopedConfig(db, {
- type: Configs.SETTINGS,
+ type: ConfigType.SETTINGS,
})
let callbackUrl = `/api/global/auth`
diff --git a/packages/backend-core/src/middleware/tenancy.js b/packages/backend-core/src/middleware/tenancy.js
deleted file mode 100644
index 8083322b29..0000000000
--- a/packages/backend-core/src/middleware/tenancy.js
+++ /dev/null
@@ -1,52 +0,0 @@
-const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy")
-const { buildMatcherRegex, matches } = require("./matchers")
-const { Headers } = require("../constants")
-
-const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => {
- // exit early if not multi-tenant
- if (!isMultiTenant()) {
- return DEFAULT_TENANT_ID
- }
-
- let tenantId
- const allowQs = opts && opts.allowQs
- const allowNoTenant = opts && opts.allowNoTenant
- const header = ctx.request.headers[Headers.TENANT_ID]
- const user = ctx.user || {}
- if (allowQs) {
- const query = ctx.request.query || {}
- tenantId = query.tenantId
- }
- // override query string (if allowed) by user, or header
- // URL params cannot be used in a middleware, as they are
- // processed later in the chain
- tenantId = user.tenantId || header || tenantId
-
- // Set the tenantId from the subdomain
- if (!tenantId) {
- tenantId = ctx.subdomains && ctx.subdomains[0]
- }
-
- if (!tenantId && !allowNoTenant) {
- ctx.throw(403, "Tenant id not set")
- }
-
- return tenantId
-}
-
-module.exports = (
- allowQueryStringPatterns,
- noTenancyPatterns,
- opts = { noTenancyRequired: false }
-) => {
- const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
- const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
-
- return async function (ctx, next) {
- const allowNoTenant =
- opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
- const allowQs = !!matches(ctx, allowQsOptions)
- const tenantId = getTenantID(ctx, { allowQs, allowNoTenant })
- return doInTenant(tenantId, next)
- }
-}
diff --git a/packages/backend-core/src/middleware/tenancy.ts b/packages/backend-core/src/middleware/tenancy.ts
new file mode 100644
index 0000000000..78da2bb3e8
--- /dev/null
+++ b/packages/backend-core/src/middleware/tenancy.ts
@@ -0,0 +1,35 @@
+import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { buildMatcherRegex, matches } from "./matchers"
+import { Header } from "../constants"
+import {
+ BBContext,
+ EndpointMatcher,
+ GetTenantIdOptions,
+ TenantResolutionStrategy,
+} from "@budibase/types"
+
+export = function (
+ allowQueryStringPatterns: EndpointMatcher[],
+ noTenancyPatterns: EndpointMatcher[],
+ opts: { noTenancyRequired?: boolean } = { noTenancyRequired: false }
+) {
+ const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
+ const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
+
+ return async function (ctx: BBContext | any, next: any) {
+ const allowNoTenant =
+ opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
+ const tenantOpts: GetTenantIdOptions = {
+ allowNoTenant,
+ }
+
+ const allowQs = !!matches(ctx, allowQsOptions)
+ if (!allowQs) {
+ tenantOpts.excludeStrategies = [TenantResolutionStrategy.QUERY]
+ }
+
+ const tenantId = getTenantIDFromCtx(ctx, tenantOpts)
+ ctx.set(Header.TENANT_ID, tenantId as string)
+ return doInTenant(tenantId, next)
+ }
+}
diff --git a/packages/backend-core/src/middleware/tests/matchers.spec.ts b/packages/backend-core/src/middleware/tests/matchers.spec.ts
new file mode 100644
index 0000000000..c39bbb6dd3
--- /dev/null
+++ b/packages/backend-core/src/middleware/tests/matchers.spec.ts
@@ -0,0 +1,134 @@
+import * as matchers from "../matchers"
+import { structures } from "../../../tests"
+
+describe("matchers", () => {
+ it("matches by path and method", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "POST",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests"
+ ctx.request.method = "POST"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(true)
+ })
+
+ it("wildcards path", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "POST",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests/id/something/else"
+ ctx.request.method = "POST"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(true)
+ })
+
+ it("doesn't wildcard path with strict", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "POST",
+ strict: true,
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests/id/something/else"
+ ctx.request.method = "POST"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(false)
+ })
+
+ it("matches with param", () => {
+ const pattern = [
+ {
+ route: "/api/tests/:testId",
+ method: "GET",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests/id"
+ ctx.request.method = "GET"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(true)
+ })
+
+ // TODO: Support the below behaviour
+ // Strict does not work when a param is present
+ // it("matches with param with strict", () => {
+ // const pattern = [{
+ // route: "/api/tests/:testId",
+ // method: "GET",
+ // strict: true
+ // }]
+ // const ctx = structures.koa.newContext()
+ // ctx.request.url = "/api/tests/id"
+ // ctx.request.method = "GET"
+ //
+ // const built = matchers.buildMatcherRegex(pattern)
+ //
+ // expect(!!matchers.matches(ctx, built)).toBe(true)
+ // })
+
+ it("doesn't match by path", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "POST",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/unknown"
+ ctx.request.method = "POST"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(false)
+ })
+
+ it("doesn't match by method", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "POST",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests"
+ ctx.request.method = "GET"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(false)
+ })
+
+ it("matches by path and wildcard method", () => {
+ const pattern = [
+ {
+ route: "/api/tests",
+ method: "ALL",
+ },
+ ]
+ const ctx = structures.koa.newContext()
+ ctx.request.url = "/api/tests"
+ ctx.request.method = "GET"
+
+ const built = matchers.buildMatcherRegex(pattern)
+
+ expect(!!matchers.matches(ctx, built)).toBe(true)
+ })
+})
diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts
index 34ec0f0cad..6eba56ab43 100644
--- a/packages/backend-core/src/migrations/definitions.ts
+++ b/packages/backend-core/src/migrations/definitions.ts
@@ -11,20 +11,12 @@ export const DEFINITIONS: MigrationDefinition[] = [
},
{
type: MigrationType.GLOBAL,
- name: MigrationName.QUOTAS_1,
+ name: MigrationName.SYNC_QUOTAS,
},
{
type: MigrationType.APP,
name: MigrationName.APP_URLS,
},
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.DEVELOPER_QUOTA,
- },
- {
- type: MigrationType.GLOBAL,
- name: MigrationName.PUBLISHED_APP_QUOTA,
- },
{
type: MigrationType.APP,
name: MigrationName.EVENT_APP_BACKFILL,
diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts
index 2e4ef0da76..7bc2dec290 100644
--- a/packages/backend-core/src/migrations/migrations.ts
+++ b/packages/backend-core/src/migrations/migrations.ts
@@ -1,14 +1,13 @@
import { DEFAULT_TENANT_ID } from "../constants"
-import { doWithDB } from "../db"
-import { DocumentTypes, StaticDatabases } from "../db/constants"
-import { getAllApps } from "../db/utils"
-import environment from "../environment"
import {
- doInTenant,
- getTenantIds,
+ DocumentType,
+ StaticDatabases,
+ getAllApps,
getGlobalDBName,
- getTenantId,
-} from "../tenancy"
+ doWithDB,
+} from "../db"
+import environment from "../environment"
+import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
import * as context from "../context"
import { DEFINITIONS } from "."
import {
@@ -16,15 +15,16 @@ import {
MigrationOptions,
MigrationType,
MigrationNoOpOptions,
+ App,
} from "@budibase/types"
export const getMigrationsDoc = async (db: any) => {
// get the migrations doc
try {
- return await db.get(DocumentTypes.MIGRATIONS)
+ return await db.get(DocumentType.MIGRATIONS)
} catch (err: any) {
if (err.status && err.status === 404) {
- return { _id: DocumentTypes.MIGRATIONS }
+ return { _id: DocumentType.MIGRATIONS }
} else {
console.error(err)
throw err
@@ -45,7 +45,7 @@ export const runMigration = async (
options: MigrationOptions = {}
) => {
const migrationType = migration.type
- let tenantId: string
+ let tenantId: string | undefined
if (migrationType !== MigrationType.INSTALLATION) {
tenantId = getTenantId()
}
@@ -59,14 +59,17 @@ export const runMigration = async (
}
// get the db to store the migration in
- let dbNames
+ let dbNames: string[]
if (migrationType === MigrationType.GLOBAL) {
dbNames = [getGlobalDBName()]
} else if (migrationType === MigrationType.APP) {
if (options.noOp) {
+ if (!options.noOp.appId) {
+ throw new Error("appId is required for noOp app migration")
+ }
dbNames = [options.noOp.appId]
} else {
- const apps = await getAllApps(migration.appOpts)
+ const apps = (await getAllApps(migration.appOpts)) as App[]
dbNames = apps.map(app => app.appId)
}
} else if (migrationType === MigrationType.INSTALLATION) {
diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js
index c5ec143143..b7d2e14ea5 100644
--- a/packages/backend-core/src/migrations/tests/index.spec.js
+++ b/packages/backend-core/src/migrations/tests/index.spec.js
@@ -1,9 +1,9 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
const { runMigrations, getMigrationsDoc } = require("../index")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
const {
StaticDatabases,
-} = require("../../db/utils")
+} = require("../../constants")
let db
@@ -18,7 +18,7 @@ describe("migrations", () => {
}]
beforeEach(() => {
- db = dangerousGetDB(StaticDatabases.GLOBAL.name)
+ db = getDB(StaticDatabases.GLOBAL.name)
})
afterEach(async () => {
diff --git a/packages/backend-core/src/newid.ts b/packages/backend-core/src/newid.ts
new file mode 100644
index 0000000000..5676c23f48
--- /dev/null
+++ b/packages/backend-core/src/newid.ts
@@ -0,0 +1,5 @@
+import { v4 } from "uuid"
+
+export function newid() {
+ return v4().replace(/-/g, "")
+}
diff --git a/packages/backend-core/src/objectStore/index.ts b/packages/backend-core/src/objectStore/index.ts
index 503ab9bca0..2971834f0e 100644
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@@ -1,325 +1,2 @@
-const sanitize = require("sanitize-s3-objectkey")
-import AWS from "aws-sdk"
-import stream from "stream"
-import fetch from "node-fetch"
-import tar from "tar-fs"
-const zlib = require("zlib")
-import { promisify } from "util"
-import { join } from "path"
-import fs from "fs"
-import env from "../environment"
-import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
-import { v4 } from "uuid"
-import { APP_PREFIX, APP_DEV_PREFIX } from "../db/utils"
-
-const streamPipeline = promisify(stream.pipeline)
-// use this as a temporary store of buckets that are being created
-const STATE = {
- bucketCreationPromises: {},
-}
-
-const CONTENT_TYPE_MAP: any = {
- html: "text/html",
- css: "text/css",
- js: "application/javascript",
- json: "application/json",
-}
-const STRING_CONTENT_TYPES = [
- CONTENT_TYPE_MAP.html,
- CONTENT_TYPE_MAP.css,
- CONTENT_TYPE_MAP.js,
- CONTENT_TYPE_MAP.json,
-]
-
-// does normal sanitization and then swaps dev apps to apps
-export function sanitizeKey(input: any) {
- return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
-}
-
-// simply handles the dev app to app conversion
-export function sanitizeBucket(input: any) {
- return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
-}
-
-function publicPolicy(bucketName: any) {
- return {
- Version: "2012-10-17",
- Statement: [
- {
- Effect: "Allow",
- Principal: {
- AWS: ["*"],
- },
- Action: "s3:GetObject",
- Resource: [`arn:aws:s3:::${bucketName}/*`],
- },
- ],
- }
-}
-
-const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS, ObjectStoreBuckets.GLOBAL]
-
-/**
- * Gets a connection to the object store using the S3 SDK.
- * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
- * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
- * @constructor
- */
-export const ObjectStore = (bucket: any) => {
- AWS.config.update({
- accessKeyId: env.MINIO_ACCESS_KEY,
- secretAccessKey: env.MINIO_SECRET_KEY,
- region: env.AWS_REGION,
- })
- const config: any = {
- s3ForcePathStyle: true,
- signatureVersion: "v4",
- apiVersion: "2006-03-01",
- }
- if (bucket) {
- config.params = {
- Bucket: sanitizeBucket(bucket),
- }
- }
- if (env.MINIO_URL) {
- config.endpoint = env.MINIO_URL
- }
- return new AWS.S3(config)
-}
-
-/**
- * Given an object store and a bucket name this will make sure the bucket exists,
- * if it does not exist then it will create it.
- */
-export const makeSureBucketExists = async (client: any, bucketName: any) => {
- bucketName = sanitizeBucket(bucketName)
- try {
- await client
- .headBucket({
- Bucket: bucketName,
- })
- .promise()
- } catch (err: any) {
- const promises: any = STATE.bucketCreationPromises
- const doesntExist = err.statusCode === 404,
- noAccess = err.statusCode === 403
- if (promises[bucketName]) {
- await promises[bucketName]
- } else if (doesntExist || noAccess) {
- if (doesntExist) {
- // bucket doesn't exist create it
- promises[bucketName] = client
- .createBucket({
- Bucket: bucketName,
- })
- .promise()
- await promises[bucketName]
- delete promises[bucketName]
- }
- // public buckets are quite hidden in the system, make sure
- // no bucket is set accidentally
- if (PUBLIC_BUCKETS.includes(bucketName)) {
- await client
- .putBucketPolicy({
- Bucket: bucketName,
- Policy: JSON.stringify(publicPolicy(bucketName)),
- })
- .promise()
- }
- } else {
- throw new Error("Unable to write to object store bucket.")
- }
- }
-}
-
-/**
- * Uploads the contents of a file given the required parameters, useful when
- * temp files in use (for example file uploaded as an attachment).
- */
-export const upload = async ({
- bucket: bucketName,
- filename,
- path,
- type,
- metadata,
-}: any) => {
- const extension = [...filename.split(".")].pop()
- const fileBytes = fs.readFileSync(path)
-
- const objectStore = ObjectStore(bucketName)
- await makeSureBucketExists(objectStore, bucketName)
-
- const config: any = {
- // windows file paths need to be converted to forward slashes for s3
- Key: sanitizeKey(filename),
- Body: fileBytes,
- ContentType: type || CONTENT_TYPE_MAP[extension.toLowerCase()],
- }
- if (metadata) {
- config.Metadata = metadata
- }
- return objectStore.upload(config).promise()
-}
-
-/**
- * Similar to the upload function but can be used to send a file stream
- * through to the object store.
- */
-export const streamUpload = async (
- bucketName: any,
- filename: any,
- stream: any,
- extra = {}
-) => {
- const objectStore = ObjectStore(bucketName)
- await makeSureBucketExists(objectStore, bucketName)
-
- const params = {
- Bucket: sanitizeBucket(bucketName),
- Key: sanitizeKey(filename),
- Body: stream,
- ...extra,
- }
- return objectStore.upload(params).promise()
-}
-
-/**
- * retrieves the contents of a file from the object store, if it is a known content type it
- * will be converted, otherwise it will be returned as a buffer stream.
- */
-export const retrieve = async (bucketName: any, filepath: any) => {
- const objectStore = ObjectStore(bucketName)
- const params = {
- Bucket: sanitizeBucket(bucketName),
- Key: sanitizeKey(filepath),
- }
- const response: any = await objectStore.getObject(params).promise()
- // currently these are all strings
- if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
- return response.Body.toString("utf8")
- } else {
- return response.Body
- }
-}
-
-/**
- * Same as retrieval function but puts to a temporary file.
- */
-export const retrieveToTmp = async (bucketName: any, filepath: any) => {
- bucketName = sanitizeBucket(bucketName)
- filepath = sanitizeKey(filepath)
- const data = await retrieve(bucketName, filepath)
- const outputPath = join(budibaseTempDir(), v4())
- fs.writeFileSync(outputPath, data)
- return outputPath
-}
-
-/**
- * Delete a single file.
- */
-export const deleteFile = async (bucketName: any, filepath: any) => {
- const objectStore = ObjectStore(bucketName)
- await makeSureBucketExists(objectStore, bucketName)
- const params = {
- Bucket: bucketName,
- Key: filepath,
- }
- return objectStore.deleteObject(params)
-}
-
-export const deleteFiles = async (bucketName: any, filepaths: any) => {
- const objectStore = ObjectStore(bucketName)
- await makeSureBucketExists(objectStore, bucketName)
- const params = {
- Bucket: bucketName,
- Delete: {
- Objects: filepaths.map((path: any) => ({ Key: path })),
- },
- }
- return objectStore.deleteObjects(params).promise()
-}
-
-/**
- * Delete a path, including everything within.
- */
-export const deleteFolder = async (
- bucketName: any,
- folder: any
-): Promise => {
- bucketName = sanitizeBucket(bucketName)
- folder = sanitizeKey(folder)
- const client = ObjectStore(bucketName)
- const listParams = {
- Bucket: bucketName,
- Prefix: folder,
- }
-
- let response: any = await client.listObjects(listParams).promise()
- if (response.Contents.length === 0) {
- return
- }
- const deleteParams: any = {
- Bucket: bucketName,
- Delete: {
- Objects: [],
- },
- }
-
- response.Contents.forEach((content: any) => {
- deleteParams.Delete.Objects.push({ Key: content.Key })
- })
-
- response = await client.deleteObjects(deleteParams).promise()
- // can only empty 1000 items at once
- if (response.Deleted.length === 1000) {
- return deleteFolder(bucketName, folder)
- }
-}
-
-export const uploadDirectory = async (
- bucketName: any,
- localPath: any,
- bucketPath: any
-) => {
- bucketName = sanitizeBucket(bucketName)
- let uploads = []
- const files = fs.readdirSync(localPath, { withFileTypes: true })
- for (let file of files) {
- const path = sanitizeKey(join(bucketPath, file.name))
- const local = join(localPath, file.name)
- if (file.isDirectory()) {
- uploads.push(uploadDirectory(bucketName, local, path))
- } else {
- uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
- }
- }
- await Promise.all(uploads)
- return files
-}
-
-exports.downloadTarballDirect = async (url: string, path: string) => {
- path = sanitizeKey(path)
- const response = await fetch(url)
- if (!response.ok) {
- throw new Error(`unexpected response ${response.statusText}`)
- }
-
- await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
-}
-
-export const downloadTarball = async (url: any, bucketName: any, path: any) => {
- bucketName = sanitizeBucket(bucketName)
- path = sanitizeKey(path)
- const response = await fetch(url)
- if (!response.ok) {
- throw new Error(`unexpected response ${response.statusText}`)
- }
-
- const tmpPath = join(budibaseTempDir(), path)
- await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
- if (!env.isTest() && env.SELF_HOSTED) {
- await uploadDirectory(bucketName, tmpPath, path)
- }
- // return the temporary path incase there is a use for it
- return tmpPath
-}
+export * from "./objectStore"
+export * from "./utils"
diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts
new file mode 100644
index 0000000000..2ae8848c53
--- /dev/null
+++ b/packages/backend-core/src/objectStore/objectStore.ts
@@ -0,0 +1,426 @@
+const sanitize = require("sanitize-s3-objectkey")
+import AWS from "aws-sdk"
+import stream from "stream"
+import fetch from "node-fetch"
+import tar from "tar-fs"
+const zlib = require("zlib")
+import { promisify } from "util"
+import { join } from "path"
+import fs from "fs"
+import env from "../environment"
+import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
+import { v4 } from "uuid"
+import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
+
+const streamPipeline = promisify(stream.pipeline)
+// use this as a temporary store of buckets that are being created
+const STATE = {
+ bucketCreationPromises: {},
+}
+
+type ListParams = {
+ ContinuationToken?: string
+}
+
+type UploadParams = {
+ bucket: string
+ filename: string
+ path: string
+ type?: string
+ // can be undefined, we will remove it
+ metadata?: {
+ [key: string]: string | undefined
+ }
+}
+
+const CONTENT_TYPE_MAP: any = {
+ txt: "text/plain",
+ html: "text/html",
+ css: "text/css",
+ js: "application/javascript",
+ json: "application/json",
+ gz: "application/gzip",
+}
+const STRING_CONTENT_TYPES = [
+ CONTENT_TYPE_MAP.html,
+ CONTENT_TYPE_MAP.css,
+ CONTENT_TYPE_MAP.js,
+ CONTENT_TYPE_MAP.json,
+]
+
+// does normal sanitization and then swaps dev apps to apps
+export function sanitizeKey(input: string) {
+ return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
+}
+
+// simply handles the dev app to app conversion
+export function sanitizeBucket(input: string) {
+ return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
+}
+
+function publicPolicy(bucketName: string) {
+ return {
+ Version: "2012-10-17",
+ Statement: [
+ {
+ Effect: "Allow",
+ Principal: {
+ AWS: ["*"],
+ },
+ Action: "s3:GetObject",
+ Resource: [`arn:aws:s3:::${bucketName}/*`],
+ },
+ ],
+ }
+}
+
+const PUBLIC_BUCKETS = [
+ ObjectStoreBuckets.APPS,
+ ObjectStoreBuckets.GLOBAL,
+ ObjectStoreBuckets.PLUGINS,
+]
+
+/**
+ * Gets a connection to the object store using the S3 SDK.
+ * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
+ * @constructor
+ */
+export const ObjectStore = (bucket: string) => {
+ const config: any = {
+ s3ForcePathStyle: true,
+ signatureVersion: "v4",
+ apiVersion: "2006-03-01",
+ accessKeyId: env.MINIO_ACCESS_KEY,
+ secretAccessKey: env.MINIO_SECRET_KEY,
+ region: env.AWS_REGION,
+ }
+ if (bucket) {
+ config.params = {
+ Bucket: sanitizeBucket(bucket),
+ }
+ }
+ if (env.MINIO_URL) {
+ config.endpoint = env.MINIO_URL
+ }
+ return new AWS.S3(config)
+}
+
+/**
+ * Given an object store and a bucket name this will make sure the bucket exists,
+ * if it does not exist then it will create it.
+ */
+export const makeSureBucketExists = async (client: any, bucketName: string) => {
+ bucketName = sanitizeBucket(bucketName)
+ try {
+ await client
+ .headBucket({
+ Bucket: bucketName,
+ })
+ .promise()
+ } catch (err: any) {
+ const promises: any = STATE.bucketCreationPromises
+ const doesntExist = err.statusCode === 404,
+ noAccess = err.statusCode === 403
+ if (promises[bucketName]) {
+ await promises[bucketName]
+ } else if (doesntExist || noAccess) {
+ if (doesntExist) {
+ // bucket doesn't exist create it
+ promises[bucketName] = client
+ .createBucket({
+ Bucket: bucketName,
+ })
+ .promise()
+ await promises[bucketName]
+ delete promises[bucketName]
+ }
+ // public buckets are quite hidden in the system, make sure
+ // no bucket is set accidentally
+ if (PUBLIC_BUCKETS.includes(bucketName)) {
+ await client
+ .putBucketPolicy({
+ Bucket: bucketName,
+ Policy: JSON.stringify(publicPolicy(bucketName)),
+ })
+ .promise()
+ }
+ } else {
+ throw new Error("Unable to write to object store bucket.")
+ }
+ }
+}
+
+/**
+ * Uploads the contents of a file given the required parameters, useful when
+ * temp files in use (for example file uploaded as an attachment).
+ */
+export const upload = async ({
+ bucket: bucketName,
+ filename,
+ path,
+ type,
+ metadata,
+}: UploadParams) => {
+ const extension = filename.split(".").pop()
+ const fileBytes = fs.readFileSync(path)
+
+ const objectStore = ObjectStore(bucketName)
+ await makeSureBucketExists(objectStore, bucketName)
+
+ let contentType = type
+ if (!contentType) {
+ contentType = extension
+ ? CONTENT_TYPE_MAP[extension.toLowerCase()]
+ : CONTENT_TYPE_MAP.txt
+ }
+ const config: any = {
+ // windows file paths need to be converted to forward slashes for s3
+ Key: sanitizeKey(filename),
+ Body: fileBytes,
+ ContentType: contentType,
+ }
+ if (metadata && typeof metadata === "object") {
+ // remove any nullish keys from the metadata object, as these may be considered invalid
+ for (let key of Object.keys(metadata)) {
+ if (!metadata[key] || typeof metadata[key] !== "string") {
+ delete metadata[key]
+ }
+ }
+ config.Metadata = metadata
+ }
+ return objectStore.upload(config).promise()
+}
+
+/**
+ * Similar to the upload function but can be used to send a file stream
+ * through to the object store.
+ */
+export const streamUpload = async (
+ bucketName: string,
+ filename: string,
+ stream: any,
+ extra = {}
+) => {
+ const objectStore = ObjectStore(bucketName)
+ await makeSureBucketExists(objectStore, bucketName)
+
+ // Set content type for certain known extensions
+ if (filename?.endsWith(".js")) {
+ extra = {
+ ...extra,
+ ContentType: "application/javascript",
+ }
+ } else if (filename?.endsWith(".svg")) {
+ extra = {
+ ...extra,
+ ContentType: "image",
+ }
+ }
+
+ const params = {
+ Bucket: sanitizeBucket(bucketName),
+ Key: sanitizeKey(filename),
+ Body: stream,
+ ...extra,
+ }
+ return objectStore.upload(params).promise()
+}
+
+/**
+ * retrieves the contents of a file from the object store, if it is a known content type it
+ * will be converted, otherwise it will be returned as a buffer stream.
+ */
+export const retrieve = async (bucketName: string, filepath: string) => {
+ const objectStore = ObjectStore(bucketName)
+ const params = {
+ Bucket: sanitizeBucket(bucketName),
+ Key: sanitizeKey(filepath),
+ }
+ const response: any = await objectStore.getObject(params).promise()
+ // currently these are all strings
+ if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
+ return response.Body.toString("utf8")
+ } else {
+ return response.Body
+ }
+}
+
+export const listAllObjects = async (bucketName: string, path: string) => {
+ const objectStore = ObjectStore(bucketName)
+ const list = (params: ListParams = {}) => {
+ return objectStore
+ .listObjectsV2({
+ ...params,
+ Bucket: sanitizeBucket(bucketName),
+ Prefix: sanitizeKey(path),
+ })
+ .promise()
+ }
+ let isTruncated = false,
+ token,
+ objects: AWS.S3.Types.Object[] = []
+ do {
+ let params: ListParams = {}
+ if (token) {
+ params.ContinuationToken = token
+ }
+ const response = await list(params)
+ if (response.Contents) {
+ objects = objects.concat(response.Contents)
+ }
+ isTruncated = !!response.IsTruncated
+ } while (isTruncated)
+ return objects
+}
+
+/**
+ * Same as retrieval function but puts to a temporary file.
+ */
+export const retrieveToTmp = async (bucketName: string, filepath: string) => {
+ bucketName = sanitizeBucket(bucketName)
+ filepath = sanitizeKey(filepath)
+ const data = await retrieve(bucketName, filepath)
+ const outputPath = join(budibaseTempDir(), v4())
+ fs.writeFileSync(outputPath, data)
+ return outputPath
+}
+
+export const retrieveDirectory = async (bucketName: string, path: string) => {
+ let writePath = join(budibaseTempDir(), v4())
+ fs.mkdirSync(writePath)
+ const objects = await listAllObjects(bucketName, path)
+ let fullObjects = await Promise.all(
+ objects.map(obj => retrieve(bucketName, obj.Key!))
+ )
+ let count = 0
+ for (let obj of objects) {
+ const filename = obj.Key!
+ const data = fullObjects[count++]
+ const possiblePath = filename.split("/")
+ if (possiblePath.length > 1) {
+ const dirs = possiblePath.slice(0, possiblePath.length - 1)
+ fs.mkdirSync(join(writePath, ...dirs), { recursive: true })
+ }
+ fs.writeFileSync(join(writePath, ...possiblePath), data)
+ }
+ return writePath
+}
+
+/**
+ * Delete a single file.
+ */
+export const deleteFile = async (bucketName: string, filepath: string) => {
+ const objectStore = ObjectStore(bucketName)
+ await makeSureBucketExists(objectStore, bucketName)
+ const params = {
+ Bucket: bucketName,
+ Key: filepath,
+ }
+ return objectStore.deleteObject(params)
+}
+
+export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
+ const objectStore = ObjectStore(bucketName)
+ await makeSureBucketExists(objectStore, bucketName)
+ const params = {
+ Bucket: bucketName,
+ Delete: {
+ Objects: filepaths.map((path: any) => ({ Key: path })),
+ },
+ }
+ return objectStore.deleteObjects(params).promise()
+}
+
+/**
+ * Delete a path, including everything within.
+ */
+export const deleteFolder = async (
+ bucketName: string,
+ folder: string
+): Promise => {
+ bucketName = sanitizeBucket(bucketName)
+ folder = sanitizeKey(folder)
+ const client = ObjectStore(bucketName)
+ const listParams = {
+ Bucket: bucketName,
+ Prefix: folder,
+ }
+
+ let response: any = await client.listObjects(listParams).promise()
+ if (response.Contents.length === 0) {
+ return
+ }
+ const deleteParams: any = {
+ Bucket: bucketName,
+ Delete: {
+ Objects: [],
+ },
+ }
+
+ response.Contents.forEach((content: any) => {
+ deleteParams.Delete.Objects.push({ Key: content.Key })
+ })
+
+ response = await client.deleteObjects(deleteParams).promise()
+ // can only empty 1000 items at once
+ if (response.Deleted.length === 1000) {
+ return deleteFolder(bucketName, folder)
+ }
+}
+
+export const uploadDirectory = async (
+ bucketName: string,
+ localPath: string,
+ bucketPath: string
+) => {
+ bucketName = sanitizeBucket(bucketName)
+ let uploads = []
+ const files = fs.readdirSync(localPath, { withFileTypes: true })
+ for (let file of files) {
+ const path = sanitizeKey(join(bucketPath, file.name))
+ const local = join(localPath, file.name)
+ if (file.isDirectory()) {
+ uploads.push(uploadDirectory(bucketName, local, path))
+ } else {
+ uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
+ }
+ }
+ await Promise.all(uploads)
+ return files
+}
+
+export const downloadTarballDirect = async (
+ url: string,
+ path: string,
+ headers = {}
+) => {
+ path = sanitizeKey(path)
+ const response = await fetch(url, { headers })
+ if (!response.ok) {
+ throw new Error(`unexpected response ${response.statusText}`)
+ }
+
+ await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+}
+
+export const downloadTarball = async (
+ url: string,
+ bucketName: string,
+ path: string
+) => {
+ bucketName = sanitizeBucket(bucketName)
+ path = sanitizeKey(path)
+ const response = await fetch(url)
+ if (!response.ok) {
+ throw new Error(`unexpected response ${response.statusText}`)
+ }
+
+ const tmpPath = join(budibaseTempDir(), path)
+ await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
+ if (!env.isTest() && env.SELF_HOSTED) {
+ await uploadDirectory(bucketName, tmpPath, path)
+ }
+ // return the temporary path incase there is a use for it
+ return tmpPath
+}
diff --git a/packages/backend-core/src/objectStore/utils.js b/packages/backend-core/src/objectStore/utils.js
deleted file mode 100644
index a243553df8..0000000000
--- a/packages/backend-core/src/objectStore/utils.js
+++ /dev/null
@@ -1,15 +0,0 @@
-const { join } = require("path")
-const { tmpdir } = require("os")
-const env = require("../environment")
-
-exports.ObjectStoreBuckets = {
- BACKUPS: env.BACKUPS_BUCKET_NAME,
- APPS: env.APPS_BUCKET_NAME,
- TEMPLATES: env.TEMPLATES_BUCKET_NAME,
- GLOBAL: env.GLOBAL_BUCKET_NAME,
- GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
-}
-
-exports.budibaseTempDir = function () {
- return join(tmpdir(), ".budibase")
-}
diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts
new file mode 100644
index 0000000000..f3c9e93943
--- /dev/null
+++ b/packages/backend-core/src/objectStore/utils.ts
@@ -0,0 +1,28 @@
+import { join } from "path"
+import { tmpdir } from "os"
+import fs from "fs"
+import env from "../environment"
+
+/****************************************************
+ * NOTE: When adding a new bucket - name *
+ * sure that S3 usages (like budibase-infra) *
+ * have been updated to have a unique bucket name. *
+ ****************************************************/
+// can't be an enum - only numbers can be used for computed types
+export const ObjectStoreBuckets = {
+ BACKUPS: env.BACKUPS_BUCKET_NAME,
+ APPS: env.APPS_BUCKET_NAME,
+ TEMPLATES: env.TEMPLATES_BUCKET_NAME,
+ GLOBAL: env.GLOBAL_BUCKET_NAME,
+ GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
+ PLUGINS: env.PLUGIN_BUCKET_NAME,
+}
+
+const bbTmp = join(tmpdir(), ".budibase")
+if (!fs.existsSync(bbTmp)) {
+ fs.mkdirSync(bbTmp)
+}
+
+export function budibaseTempDir() {
+ return bbTmp
+}
diff --git a/packages/backend-core/src/pino.js b/packages/backend-core/src/pino.js
deleted file mode 100644
index 69962b3841..0000000000
--- a/packages/backend-core/src/pino.js
+++ /dev/null
@@ -1,11 +0,0 @@
-const env = require("./environment")
-
-exports.pinoSettings = () => ({
- prettyPrint: {
- levelFirst: true,
- },
- level: env.LOG_LEVEL || "error",
- autoLogging: {
- ignore: req => req.url.includes("/health"),
- },
-})
diff --git a/packages/backend-core/src/pino.ts b/packages/backend-core/src/pino.ts
new file mode 100644
index 0000000000..4140f428e1
--- /dev/null
+++ b/packages/backend-core/src/pino.ts
@@ -0,0 +1,13 @@
+import env from "./environment"
+
+export function pinoSettings() {
+ return {
+ prettyPrint: {
+ levelFirst: true,
+ },
+ level: env.LOG_LEVEL || "error",
+ autoLogging: {
+ ignore: (req: { url: string }) => req.url.includes("/health"),
+ },
+ }
+}
diff --git a/packages/backend-core/src/pkg/cache.ts b/packages/backend-core/src/pkg/cache.ts
deleted file mode 100644
index 1aaa40370d..0000000000
--- a/packages/backend-core/src/pkg/cache.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-import * as generic from "../cache/generic"
-import * as user from "../cache/user"
-import * as app from "../cache/appMetadata"
-
-export = {
- app,
- user,
- ...generic,
-}
diff --git a/packages/backend-core/src/pkg/context.ts b/packages/backend-core/src/pkg/context.ts
deleted file mode 100644
index 5caa82ab0c..0000000000
--- a/packages/backend-core/src/pkg/context.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-import {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
-} from "../context"
-
-import * as identity from "../context/identity"
-
-export = {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
- identity,
-}
diff --git a/packages/backend-core/src/pkg/db.ts b/packages/backend-core/src/pkg/db.ts
deleted file mode 100644
index 0254adddd5..0000000000
--- a/packages/backend-core/src/pkg/db.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export * from "../db"
-export * from "../db/utils"
-export * from "../db/views"
-export * from "../db/pouch"
-export * from "../db/constants"
diff --git a/packages/backend-core/src/pkg/objectStore.ts b/packages/backend-core/src/pkg/objectStore.ts
deleted file mode 100644
index 0447c6b3c2..0000000000
--- a/packages/backend-core/src/pkg/objectStore.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export * from "../objectStore"
-export * from "../objectStore/utils"
diff --git a/packages/backend-core/src/pkg/redis.ts b/packages/backend-core/src/pkg/redis.ts
deleted file mode 100644
index 65ab186d9a..0000000000
--- a/packages/backend-core/src/pkg/redis.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-import Client from "../redis"
-import utils from "../redis/utils"
-import clients from "../redis/init"
-
-export = {
- Client,
- utils,
- clients,
-}
diff --git a/packages/backend-core/src/pkg/utils.ts b/packages/backend-core/src/pkg/utils.ts
deleted file mode 100644
index 5272046524..0000000000
--- a/packages/backend-core/src/pkg/utils.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export * from "../utils"
-export * from "../hashing"
diff --git a/packages/backend-core/src/plugin/index.ts b/packages/backend-core/src/plugin/index.ts
new file mode 100644
index 0000000000..3eeaeaa90c
--- /dev/null
+++ b/packages/backend-core/src/plugin/index.ts
@@ -0,0 +1 @@
+export * from "./utils"
diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts
new file mode 100644
index 0000000000..7b62248bb5
--- /dev/null
+++ b/packages/backend-core/src/plugin/utils.ts
@@ -0,0 +1,96 @@
+import { DatasourceFieldType, QueryType, PluginType } from "@budibase/types"
+import joi from "joi"
+
+const DATASOURCE_TYPES = [
+ "Relational",
+ "Non-relational",
+ "Spreadsheet",
+ "Object store",
+ "Graph",
+ "API",
+]
+
+function runJoi(validator: joi.Schema, schema: any) {
+ const { error } = validator.validate(schema)
+ if (error) {
+ throw error
+ }
+}
+
+function validateComponent(schema: any) {
+ const validator = joi.object({
+ type: joi.string().allow("component").required(),
+ metadata: joi.object().unknown(true).required(),
+ hash: joi.string().optional(),
+ version: joi.string().optional(),
+ schema: joi
+ .object({
+ name: joi.string().required(),
+ settings: joi.array().items(joi.object().unknown(true)).required(),
+ })
+ .unknown(true),
+ })
+ runJoi(validator, schema)
+}
+
+function validateDatasource(schema: any) {
+ const fieldValidator = joi.object({
+ type: joi
+ .string()
+ .allow(...Object.values(DatasourceFieldType))
+ .required(),
+ required: joi.boolean().required(),
+ default: joi.any(),
+ display: joi.string(),
+ })
+
+ const queryValidator = joi
+ .object({
+ type: joi.string().allow(...Object.values(QueryType)),
+ readable: joi.boolean(),
+ fields: joi.object().pattern(joi.string(), fieldValidator),
+ })
+ .required()
+
+ const validator = joi.object({
+ type: joi.string().allow("datasource").required(),
+ metadata: joi.object().unknown(true).required(),
+ hash: joi.string().optional(),
+ version: joi.string().optional(),
+ schema: joi.object({
+ docs: joi.string(),
+ friendlyName: joi.string().required(),
+ type: joi.string().allow(...DATASOURCE_TYPES),
+ description: joi.string().required(),
+ datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
+ query: joi
+ .object()
+ .pattern(joi.string(), queryValidator)
+ .unknown(true)
+ .required(),
+ extra: joi.object().pattern(
+ joi.string(),
+ joi.object({
+ type: joi.string().required(),
+ displayName: joi.string().required(),
+ required: joi.boolean(),
+ data: joi.object(),
+ })
+ ),
+ }),
+ })
+ runJoi(validator, schema)
+}
+
+export function validate(schema: any) {
+ switch (schema?.type) {
+ case PluginType.COMPONENT:
+ validateComponent(schema)
+ break
+ case PluginType.DATASOURCE:
+ validateDatasource(schema)
+ break
+ default:
+ throw new Error(`Unknown plugin type - check schema.json: ${schema.type}`)
+ }
+}
diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts
new file mode 100644
index 0000000000..e8323dacb8
--- /dev/null
+++ b/packages/backend-core/src/queue/constants.ts
@@ -0,0 +1,4 @@
+export enum JobQueue {
+ AUTOMATION = "automationQueue",
+ APP_BACKUP = "appBackupQueue",
+}
diff --git a/packages/server/src/utilities/queue/inMemoryQueue.js b/packages/backend-core/src/queue/inMemoryQueue.ts
similarity index 82%
rename from packages/server/src/utilities/queue/inMemoryQueue.js
rename to packages/backend-core/src/queue/inMemoryQueue.ts
index 620b65cf38..acfff1c7b8 100644
--- a/packages/server/src/utilities/queue/inMemoryQueue.js
+++ b/packages/backend-core/src/queue/inMemoryQueue.ts
@@ -1,4 +1,5 @@
-let events = require("events")
+import events from "events"
+import { timeout } from "../utils"
/**
* Bull works with a Job wrapper around all messages that contains a lot more information about
@@ -9,7 +10,7 @@ let events = require("events")
* @returns {Object} A new job which can now be put onto the queue, this is mostly an
* internal structure so that an in memory queue can be easily swapped for a Bull queue.
*/
-function newJob(queue, message) {
+function newJob(queue: string, message: any) {
return {
timestamp: Date.now(),
queue: queue,
@@ -23,17 +24,25 @@ function newJob(queue, message) {
* to the consumers - in can support many inputs and many consumers.
*/
class InMemoryQueue {
+ _name: string
+ _opts?: any
+ _messages: any[]
+ _emitter: EventEmitter
+ _runCount: number
+ _addCount: number
/**
* The constructor the queue, exactly the same as that of Bulls.
* @param {string} name The name of the queue which is being configured.
* @param {object|null} opts This is not used by the in memory queue as there is no real use
* case when in memory, but is the same API as Bull
*/
- constructor(name, opts = null) {
+ constructor(name: string, opts = null) {
this._name = name
this._opts = opts
this._messages = []
this._emitter = new events.EventEmitter()
+ this._runCount = 0
+ this._addCount = 0
}
/**
@@ -45,7 +54,7 @@ class InMemoryQueue {
* note this is incredibly limited compared to Bull as in reality the Job would contain
* a lot more information about the queue and current status of Bull cluster.
*/
- process(func) {
+ process(func: any) {
this._emitter.on("message", async () => {
if (this._messages.length <= 0) {
return
@@ -55,6 +64,7 @@ class InMemoryQueue {
if (resp.then != null) {
await resp
}
+ this._runCount++
})
}
@@ -65,13 +75,15 @@ class InMemoryQueue {
* return the message to a consumer (if one is attached).
* @param {object} msg A message to be transported over the queue, this should be
* a JSON message as this is required by Bull.
+ * @param {boolean} repeat serves no purpose for the import queue.
*/
// eslint-disable-next-line no-unused-vars
- add(msg, repeat) {
+ add(msg: any, repeat: boolean) {
if (typeof msg !== "object") {
throw "Queue only supports carrying JSON."
}
this._messages.push(newJob(this._name, msg))
+ this._addCount++
this._emitter.emit("message")
}
@@ -86,7 +98,7 @@ class InMemoryQueue {
* This removes a cron which has been implemented, this is part of Bull API.
* @param {string} cronJobId The cron which is to be removed.
*/
- removeRepeatableByKey(cronJobId) {
+ removeRepeatableByKey(cronJobId: string) {
// TODO: implement for testing
console.log(cronJobId)
}
@@ -99,7 +111,7 @@ class InMemoryQueue {
}
// eslint-disable-next-line no-unused-vars
- removeJobs(pattern) {
+ removeJobs(pattern: string) {
// no-op
}
@@ -113,6 +125,16 @@ class InMemoryQueue {
async getJob() {
return {}
}
+
+ on() {
+ // do nothing
+ }
+
+ async waitForCompletion() {
+ do {
+ await timeout(50)
+ } while (this._addCount < this._runCount)
+ }
}
-module.exports = InMemoryQueue
+export = InMemoryQueue
diff --git a/packages/backend-core/src/queue/index.ts b/packages/backend-core/src/queue/index.ts
new file mode 100644
index 0000000000..b7d565ba13
--- /dev/null
+++ b/packages/backend-core/src/queue/index.ts
@@ -0,0 +1,2 @@
+export * from "./queue"
+export * from "./constants"
diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts
new file mode 100644
index 0000000000..e1975b5d06
--- /dev/null
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -0,0 +1,101 @@
+import { Job, JobId, Queue } from "bull"
+import { JobQueue } from "./constants"
+
+export type StalledFn = (job: Job) => Promise
+
+export function addListeners(
+ queue: Queue,
+ jobQueue: JobQueue,
+ removeStalledCb?: StalledFn
+) {
+ logging(queue, jobQueue)
+ if (removeStalledCb) {
+ handleStalled(queue, removeStalledCb)
+ }
+}
+
+function handleStalled(queue: Queue, removeStalledCb?: StalledFn) {
+ queue.on("stalled", async (job: Job) => {
+ if (removeStalledCb) {
+ await removeStalledCb(job)
+ } else if (job.opts.repeat) {
+ const jobId = job.id
+ const repeatJobs = await queue.getRepeatableJobs()
+ for (let repeatJob of repeatJobs) {
+ if (repeatJob.id === jobId) {
+ await queue.removeRepeatableByKey(repeatJob.key)
+ }
+ }
+ console.log(`jobId=${jobId} disabled`)
+ }
+ })
+}
+
+function logging(queue: Queue, jobQueue: JobQueue) {
+ let eventType: string
+ switch (jobQueue) {
+ case JobQueue.AUTOMATION:
+ eventType = "automation-event"
+ break
+ case JobQueue.APP_BACKUP:
+ eventType = "app-backup-event"
+ break
+ }
+ if (process.env.NODE_DEBUG?.includes("bull")) {
+ queue
+ .on("error", (error: any) => {
+ // An error occurred.
+ console.error(`${eventType}=error error=${JSON.stringify(error)}`)
+ })
+ .on("waiting", (jobId: JobId) => {
+ // A Job is waiting to be processed as soon as a worker is idling.
+ console.log(`${eventType}=waiting jobId=${jobId}`)
+ })
+ .on("active", (job: Job, jobPromise: any) => {
+ // A job has started. You can use `jobPromise.cancel()`` to abort it.
+ console.log(`${eventType}=active jobId=${job.id}`)
+ })
+ .on("stalled", (job: Job) => {
+ // A job has been marked as stalled. This is useful for debugging job
+ // workers that crash or pause the event loop.
+ console.error(
+ `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}`
+ )
+ })
+ .on("progress", (job: Job, progress: any) => {
+ // A job's progress was updated!
+ console.log(
+ `${eventType}=progress jobId=${job.id} progress=${progress}`
+ )
+ })
+ .on("completed", (job: Job, result) => {
+ // A job successfully completed with a `result`.
+ console.log(`${eventType}=completed jobId=${job.id} result=${result}`)
+ })
+ .on("failed", (job, err: any) => {
+ // A job failed with reason `err`!
+ console.log(`${eventType}=failed jobId=${job.id} error=${err}`)
+ })
+ .on("paused", () => {
+ // The queue has been paused.
+ console.log(`${eventType}=paused`)
+ })
+ .on("resumed", (job: Job) => {
+ // The queue has been resumed.
+ console.log(`${eventType}=paused jobId=${job.id}`)
+ })
+ .on("cleaned", (jobs: Job[], type: string) => {
+ // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned
+ // jobs, and `type` is the type of jobs cleaned.
+ console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`)
+ })
+ .on("drained", () => {
+ // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed)
+ console.log(`${eventType}=drained`)
+ })
+ .on("removed", (job: Job) => {
+ // A job successfully removed.
+ console.log(`${eventType}=removed jobId=${job.id}`)
+ })
+ }
+}
diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts
new file mode 100644
index 0000000000..b34d46e463
--- /dev/null
+++ b/packages/backend-core/src/queue/queue.ts
@@ -0,0 +1,51 @@
+import env from "../environment"
+import { getRedisOptions } from "../redis/utils"
+import { JobQueue } from "./constants"
+import InMemoryQueue from "./inMemoryQueue"
+import BullQueue from "bull"
+import { addListeners, StalledFn } from "./listeners"
+const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
+
+const CLEANUP_PERIOD_MS = 60 * 1000
+let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
+let cleanupInterval: NodeJS.Timeout
+
+async function cleanup() {
+ for (let queue of QUEUES) {
+ await queue.clean(CLEANUP_PERIOD_MS, "completed")
+ }
+}
+
+export function createQueue(
+ jobQueue: JobQueue,
+ opts: { removeStalledCb?: StalledFn } = {}
+): BullQueue.Queue {
+ const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
+ let queue: any
+ if (!env.isTest()) {
+ queue = new BullQueue(jobQueue, queueConfig)
+ } else {
+ queue = new InMemoryQueue(jobQueue, queueConfig)
+ }
+ addListeners(queue, jobQueue, opts?.removeStalledCb)
+ QUEUES.push(queue)
+ if (!cleanupInterval) {
+ cleanupInterval = setInterval(cleanup, CLEANUP_PERIOD_MS)
+ // fire off an initial cleanup
+ cleanup().catch(err => {
+ console.error(`Unable to cleanup automation queue initially - ${err}`)
+ })
+ }
+ return queue
+}
+
+export async function shutdown() {
+ if (QUEUES.length) {
+ clearInterval(cleanupInterval)
+ for (let queue of QUEUES) {
+ await queue.close()
+ }
+ QUEUES = []
+ }
+ console.log("Queues shutdown")
+}
diff --git a/packages/backend-core/src/redis/index.ts b/packages/backend-core/src/redis/index.ts
index 206110366f..ea4379f048 100644
--- a/packages/backend-core/src/redis/index.ts
+++ b/packages/backend-core/src/redis/index.ts
@@ -1,250 +1,6 @@
-import RedisWrapper from "../redis"
-const env = require("../environment")
-// ioredis mock is all in memory
-const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
-const {
- addDbPrefix,
- removeDbPrefix,
- getRedisOptions,
- SEPARATOR,
- SelectableDatabases,
-} = require("./utils")
-
-const RETRY_PERIOD_MS = 2000
-const STARTUP_TIMEOUT_MS = 5000
-const CLUSTERED = false
-const DEFAULT_SELECT_DB = SelectableDatabases.DEFAULT
-
-// for testing just generate the client once
-let CLOSED = false
-let CLIENTS: { [key: number]: any } = {}
-// if in test always connected
-let CONNECTED = env.isTest()
-
-function pickClient(selectDb: number): any {
- return CLIENTS[selectDb]
-}
-
-function connectionError(
- selectDb: number,
- timeout: NodeJS.Timeout,
- err: Error | string
-) {
- // manually shut down, ignore errors
- if (CLOSED) {
- return
- }
- pickClient(selectDb).disconnect()
- CLOSED = true
- // always clear this on error
- clearTimeout(timeout)
- CONNECTED = false
- console.error("Redis connection failed - " + err)
- setTimeout(() => {
- init()
- }, RETRY_PERIOD_MS)
-}
-
-/**
- * Inits the system, will error if unable to connect to redis cluster (may take up to 10 seconds) otherwise
- * will return the ioredis client which will be ready to use.
- */
-function init(selectDb = DEFAULT_SELECT_DB) {
- let timeout: NodeJS.Timeout
- CLOSED = false
- let client = pickClient(selectDb)
- // already connected, ignore
- if (client && CONNECTED) {
- return
- }
- // testing uses a single in memory client
- if (env.isTest()) {
- CLIENTS[selectDb] = new Redis(getRedisOptions())
- }
- // start the timer - only allowed 5 seconds to connect
- timeout = setTimeout(() => {
- if (!CONNECTED) {
- connectionError(
- selectDb,
- timeout,
- "Did not successfully connect in timeout"
- )
- }
- }, STARTUP_TIMEOUT_MS)
-
- // disconnect any lingering client
- if (client) {
- client.disconnect()
- }
- const { redisProtocolUrl, opts, host, port } = getRedisOptions(CLUSTERED)
-
- if (CLUSTERED) {
- client = new Redis.Cluster([{ host, port }], opts)
- } else if (redisProtocolUrl) {
- client = new Redis(redisProtocolUrl)
- } else {
- client = new Redis(opts)
- }
- // attach handlers
- client.on("end", (err: Error) => {
- connectionError(selectDb, timeout, err)
- })
- client.on("error", (err: Error) => {
- connectionError(selectDb, timeout, err)
- })
- client.on("connect", () => {
- clearTimeout(timeout)
- CONNECTED = true
- })
- CLIENTS[selectDb] = client
-}
-
-function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
- return new Promise(resolve => {
- if (pickClient(selectDb) == null) {
- init()
- } else if (CONNECTED) {
- resolve("")
- return
- }
- // check if the connection is ready
- const interval = setInterval(() => {
- if (CONNECTED) {
- clearInterval(interval)
- resolve("")
- }
- }, 500)
- })
-}
-
-/**
- * Utility function, takes a redis stream and converts it to a promisified response -
- * this can only be done with redis streams because they will have an end.
- * @param stream A redis stream, specifically as this type of stream will have an end.
- * @param client The client to use for further lookups.
- * @return {Promise
diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index fd46e77647..72d9fc93a9 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -60,8 +60,6 @@ spec:
secretKeyRef:
name: {{ template "budibase.fullname" . }}
key: jwtSecret
- - name: LOG_LEVEL
- value: {{ .Values.services.apps.logLevel | default "info" | quote }}
{{ if .Values.services.objectStore.region }}
- name: AWS_REGION
value: {{ .Values.services.objectStore.region }}
@@ -78,6 +76,14 @@ spec:
key: objectStoreSecret
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
+ - name: PLUGIN_BUCKET_NAME
+ value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+ - name: APPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.appsBucketName | quote }}
+ - name: GLOBAL_CLOUD_BUCKET_NAME
+ value: {{ .Values.services.objectStore.globalBucketName | quote }}
+ - name: BACKUPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.apps.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}
@@ -124,12 +130,34 @@ spec:
value: {{ .Values.globals.tenantFeatureFlags | quote }}
{{ if .Values.globals.bbAdminUserEmail }}
- name: BB_ADMIN_USER_EMAIL
- value: { { .Values.globals.bbAdminUserEmail | quote } }
+ value: {{ .Values.globals.bbAdminUserEmail | quote }}
{{ end }}
{{ if .Values.globals.bbAdminUserPassword }}
- name: BB_ADMIN_USER_PASSWORD
- value: { { .Values.globals.bbAdminUserPassword | quote } }
+ value: {{ .Values.globals.bbAdminUserPassword | quote }}
{{ end }}
+ {{ if .Values.globals.pluginsDir }}
+ - name: PLUGINS_DIR
+ value: {{ .Values.globals.pluginsDir | quote }}
+ {{ end }}
+ {{ if .Values.services.apps.nodeDebug }}
+ - name: NODE_DEBUG
+ value: {{ .Values.services.apps.nodeDebug | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmEnabled }}
+ - name: ELASTIC_APM_ENABLED
+ value: {{ .Values.globals.elasticApmEnabled | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmSecretToken }}
+ - name: ELASTIC_APM_SECRET_TOKEN
+ value: {{ .Values.globals.elasticApmSecretToken | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmServerUrl }}
+ - name: ELASTIC_APM_SERVER_URL
+ value: {{ .Values.globals.elasticApmServerUrl | quote }}
+ {{ end }}
+ - name: CDN_URL
+ value: {{ .Values.globals.cdnUrl }}
image: budibase/apps:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
@@ -142,7 +170,10 @@ spec:
name: bbapps
ports:
- containerPort: {{ .Values.services.apps.port }}
- resources: {}
+ {{ with .Values.services.apps.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/templates/couchdb-backup.yaml b/charts/budibase/templates/couchdb-backup.yaml
index ae062475ce..68e5eab617 100644
--- a/charts/budibase/templates/couchdb-backup.yaml
+++ b/charts/budibase/templates/couchdb-backup.yaml
@@ -38,7 +38,10 @@ spec:
image: redgeoff/replicate-couchdb-cluster
imagePullPolicy: Always
name: couchdb-backup
- resources: {}
+ {{ with .Values.services.couchdb.backup.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml
index 103f9e3ed2..144dbe539a 100644
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@@ -56,7 +56,10 @@ spec:
name: minio-service
ports:
- containerPort: {{ .Values.services.objectStore.port }}
- resources: {}
+ {{ with .Values.services.objectStore.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
- mountPath: /data
name: minio-data
diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml
index 505a46f1e8..5588022032 100644
--- a/charts/budibase/templates/proxy-service-deployment.yaml
+++ b/charts/budibase/templates/proxy-service-deployment.yaml
@@ -30,7 +30,10 @@ spec:
name: proxy-service
ports:
- containerPort: {{ .Values.services.proxy.port }}
- resources: {}
+ {{ with .Values.services.proxy.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
{{- with .Values.affinity }}
affinity:
diff --git a/charts/budibase/templates/redis-service-deployment.yaml b/charts/budibase/templates/redis-service-deployment.yaml
index 6e09346cad..d94e4d70f8 100644
--- a/charts/budibase/templates/redis-service-deployment.yaml
+++ b/charts/budibase/templates/redis-service-deployment.yaml
@@ -35,7 +35,10 @@ spec:
name: redis-service
ports:
- containerPort: {{ .Values.services.redis.port }}
- resources: {}
+ {{ with .Values.services.redis.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
volumeMounts:
- mountPath: /data
name: redis-data
diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml
index 918dab427b..df692a0723 100644
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@@ -27,6 +27,8 @@ spec:
spec:
containers:
- env:
+ - name: BUDIBASE_ENVIRONMENT
+ value: {{ .Values.globals.budibaseEnv }}
- name: DEPLOYMENT_ENVIRONMENT
value: "kubernetes"
- name: CLUSTER_PORT
@@ -75,6 +77,14 @@ spec:
key: objectStoreSecret
- name: MINIO_URL
value: {{ .Values.services.objectStore.url }}
+ - name: PLUGIN_BUCKET_NAME
+ value: {{ .Values.services.objectStore.pluginBucketName | quote }}
+ - name: APPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.appsBucketName | quote }}
+ - name: GLOBAL_CLOUD_BUCKET_NAME
+ value: {{ .Values.services.objectStore.globalBucketName | quote }}
+ - name: BACKUPS_BUCKET_NAME
+ value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: PORT
value: {{ .Values.services.worker.port | quote }}
- name: MULTI_TENANCY
@@ -125,6 +135,21 @@ spec:
value: {{ .Values.globals.google.secret | quote }}
- name: TENANT_FEATURE_FLAGS
value: {{ .Values.globals.tenantFeatureFlags | quote }}
+ {{ if .Values.globals.elasticApmEnabled }}
+ - name: ELASTIC_APM_ENABLED
+ value: {{ .Values.globals.elasticApmEnabled | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmSecretToken }}
+ - name: ELASTIC_APM_SECRET_TOKEN
+ value: {{ .Values.globals.elasticApmSecretToken | quote }}
+ {{ end }}
+ {{ if .Values.globals.elasticApmServerUrl }}
+ - name: ELASTIC_APM_SERVER_URL
+ value: {{ .Values.globals.elasticApmServerUrl | quote }}
+ {{ end }}
+ - name: CDN_URL
+ value: {{ .Values.globals.cdnUrl }}
+
image: budibase/worker:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
livenessProbe:
@@ -136,7 +161,10 @@ spec:
name: bbworker
ports:
- containerPort: {{ .Values.services.worker.port }}
- resources: {}
+ {{ with .Values.services.worker.resources }}
+ resources:
+ {{- toYaml . | nindent 10 }}
+ {{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 2734202fff..a2a761aa86 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -60,19 +60,6 @@ ingress:
port:
number: 10000
-resources:
- {}
- # We usually recommend not to specify default resources and to leave this as a conscious
- # choice for the user. This also increases chances charts run on environments with little
- # resources, such as Minikube. If you do want to specify resources, uncomment the following
- # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
- # limits:
- # cpu: 100m
- # memory: 128Mi
- # requests:
- # cpu: 100m
- # memory: 128Mi
-
autoscaling:
enabled: false
minReplicas: 1
@@ -89,9 +76,10 @@ affinity: {}
globals:
appVersion: "latest"
budibaseEnv: PRODUCTION
+ tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS"
enableAnalytics: "1"
sentryDSN: ""
- posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS"
+ posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
logLevel: info
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
@@ -110,10 +98,15 @@ globals:
# if createSecrets is set to false, you can hard-code your secrets here
internalApiKey: ""
jwtSecret: ""
+ cdnUrl: ""
smtp:
enabled: false
+# elasticApmEnabled:
+# elasticApmSecretToken:
+# elasticApmServerUrl:
+
services:
budibaseVersion: latest
dns: cluster.local
@@ -121,15 +114,19 @@ services:
proxy:
port: 10000
replicaCount: 1
+ resources: {}
apps:
port: 4002
replicaCount: 1
logLevel: info
+ resources: {}
+# nodeDebug: "" # set the value of NODE_DEBUG
worker:
port: 4003
replicaCount: 1
+ resources: {}
couchdb:
enabled: true
@@ -143,6 +140,7 @@ services:
target: ""
# backup interval in seconds
interval: ""
+ resources: {}
redis:
enabled: true # disable if using external redis
@@ -156,6 +154,7 @@ services:
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner.
storageClass: ""
+ resources: {}
objectStore:
minio: true
@@ -172,6 +171,7 @@ services:
## If undefined (the default) or set to null, no storageClassName spec is
## set, choosing the default provisioner.
storageClass: ""
+ resources: {}
# Override values in couchDB subchart
couchdb:
diff --git a/docs/DEV-SETUP-DEBIAN.md b/docs/DEV-SETUP-DEBIAN.md
index 88a124708c..9edd8286cb 100644
--- a/docs/DEV-SETUP-DEBIAN.md
+++ b/docs/DEV-SETUP-DEBIAN.md
@@ -1,12 +1,15 @@
## Dev Environment on Debian 11
-### Install Node
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
-Budibase requires a recent version of node (14+):
+Install NVM
```
-curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
-apt -y install nodejs
-node -v
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+```
+Install Node 14
+```
+nvm install 14
```
### Install npm requirements
@@ -31,7 +34,7 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show
- Docker: 20.10.5
- Docker-Compose: 1.29.2
-- Node: v16.15.1
+- Node: v14.20.1
- Yarn: 1.22.19
- Lerna: 5.1.4
diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md
index c5990e58da..d9e2dcad6a 100644
--- a/docs/DEV-SETUP-MACOSX.md
+++ b/docs/DEV-SETUP-MACOSX.md
@@ -11,7 +11,7 @@ through brew.
### Install Node
-Budibase requires a recent version of node (14+):
+Budibase requires a recent version of node 14:
```
brew install node npm
node -v
@@ -38,7 +38,7 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show
- Docker: 20.10.14
- Docker-Compose: 2.6.0
-- Node: 18.3.0
+- Node: 14.20.1
- Yarn: 1.22.19
- Lerna: 5.1.4
@@ -59,4 +59,7 @@ The dev version will be available on port 10000 i.e.
http://127.0.0.1:10000/builder/admin
| **NOTE**: If you are working on a M1 Apple Silicon, you will need to uncomment `# platform: linux/amd64` line in
-[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
\ No newline at end of file
+[hosting/docker-compose-dev.yaml](../hosting/docker-compose.dev.yaml)
+
+### Troubleshooting
+If there are errors with the `yarn setup` command, you can try installing nvm and node 14. This is the same as the instructions for Debian 11.
diff --git a/docs/DEV-SETUP-WINDOWS.md b/docs/DEV-SETUP-WINDOWS.md
new file mode 100644
index 0000000000..c5608b7567
--- /dev/null
+++ b/docs/DEV-SETUP-WINDOWS.md
@@ -0,0 +1,81 @@
+## Dev Environment on Windows 10/11 (WSL2)
+
+
+### Install WSL with Ubuntu LTS
+
+Enable WSL 2 on Windows 10/11 for docker support.
+```
+wsl --set-default-version 2
+```
+Install Ubuntu LTS.
+```
+wsl --install Ubuntu
+```
+
+Or follow the instruction here:
+https://learn.microsoft.com/en-us/windows/wsl/install
+
+### Install Docker in windows
+Download the installer from docker and install it.
+
+Check this url for more detailed instructions:
+https://docs.docker.com/desktop/install/windows-install/
+
+You should follow the next steps from within the Ubuntu terminal.
+
+### Install NVM & Node 14
+NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating
+
+Install NVM
+```
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
+```
+Install Node 14
+```
+nvm install 14
+```
+
+
+### Install npm requirements
+
+```
+npm install -g yarn jest lerna
+```
+
+### Clone the repo
+```
+git clone https://github.com/Budibase/budibase.git
+```
+
+### Check Versions
+
+This setup process was tested on Windows 11 with version numbers show below. Your mileage may vary using anything else.
+
+- Docker: 20.10.7
+- Docker-Compose: 2.10.2
+- Node: v14.20.1
+- Yarn: 1.22.19
+- Lerna: 5.5.4
+
+### Build
+
+```
+cd budibase
+yarn setup
+```
+The yarn setup command runs several build steps i.e.
+```
+node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev
+```
+So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose.
+
+The dev version will be available on port 10000 i.e.
+
+http://127.0.0.1:10000/builder/admin
+
+### Working with the code
+Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine.
+
+https://code.visualstudio.com/docs/remote/wsl
+
+Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows.
\ No newline at end of file
diff --git a/examples/nextjs-api-sales/definitions/openapi.ts b/examples/nextjs-api-sales/definitions/openapi.ts
index 4f4ad45fc6..7f7f6befec 100644
--- a/examples/nextjs-api-sales/definitions/openapi.ts
+++ b/examples/nextjs-api-sales/definitions/openapi.ts
@@ -348,7 +348,7 @@ export interface paths {
}
}
responses: {
- /** Returns the created table, including the ID which has been generated for it. This can be internal or external data sources. */
+ /** Returns the created table, including the ID which has been generated for it. This can be internal or external datasources. */
200: {
content: {
"application/json": components["schemas"]["tableOutput"]
@@ -959,7 +959,7 @@ export interface components {
query: {
/** @description The ID of the query. */
_id: string
- /** @description The ID of the data source the query belongs to. */
+ /** @description The ID of the datasource the query belongs to. */
datasourceId?: string
/** @description The bindings which are required to perform this query. */
parameters?: string[]
@@ -983,7 +983,7 @@ export interface components {
data: {
/** @description The ID of the query. */
_id: string
- /** @description The ID of the data source the query belongs to. */
+ /** @description The ID of the datasource the query belongs to. */
datasourceId?: string
/** @description The bindings which are required to perform this query. */
parameters?: string[]
diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json
index 6d75c85f01..41ce52e952 100644
--- a/examples/nextjs-api-sales/package.json
+++ b/examples/nextjs-api-sales/package.json
@@ -11,8 +11,8 @@
"dependencies": {
"bulma": "^0.9.3",
"next": "12.1.0",
- "node-fetch": "^3.2.2",
- "node-sass": "^7.0.1",
+ "node-fetch": "^3.2.10",
+ "sass": "^1.52.3",
"react": "17.0.2",
"react-dom": "17.0.2",
"react-notifications-component": "^3.4.1"
@@ -24,4 +24,4 @@
"eslint-config-next": "12.1.0",
"typescript": "4.6.2"
}
-}
+}
\ No newline at end of file
diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock
index 52c89967b2..f47fb84e33 100644
--- a/examples/nextjs-api-sales/yarn.lock
+++ b/examples/nextjs-api-sales/yarn.lock
@@ -2020,10 +2020,10 @@ node-domexception@^1.0.0:
resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
-node-fetch@^3.2.2:
- version "3.2.2"
- resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.2.tgz#16d33fbe32ca7c6ca1ca8ba5dfea1dd885c59f04"
- integrity sha512-Cwhq1JFIoon15wcIkFzubVNFE5GvXGV82pKf4knXXjvGmn7RJKcypeuqcVNZMGDZsAFWyIRya/anwAJr7TWJ7w==
+node-fetch@^3.2.10:
+ version "3.2.10"
+ resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.10.tgz#e8347f94b54ae18b57c9c049ef641cef398a85c8"
+ integrity sha512-MhuzNwdURnZ1Cp4XTazr69K0BTizsBroX7Zx3UgDSVcZYKF/6p0CBe4EUb/hLqmzVhl0UpYfgRljQ4yxE+iCxA==
dependencies:
data-uri-to-buffer "^4.0.0"
fetch-blob "^3.1.4"
diff --git a/hosting/.env b/hosting/.env
index 11dd661bf1..c5638a266f 100644
--- a/hosting/.env
+++ b/hosting/.env
@@ -22,4 +22,7 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
# An admin user can be automatically created initially if these are set
BB_ADMIN_USER_EMAIL=
-BB_ADMIN_USER_PASSWORD=
\ No newline at end of file
+BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=
\ No newline at end of file
diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml
index f669f9261d..5b2adc2665 100644
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@@ -25,9 +25,12 @@ services:
REDIS_PASSWORD: ${REDIS_PASSWORD}
BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+ PLUGINS_DIR: ${PLUGINS_DIR}
depends_on:
- worker-service
- redis-service
+# volumes:
+# - /some/path/to/plugins:/plugins
worker-service:
restart: unless-stopped
@@ -76,6 +79,9 @@ services:
- "${MAIN_PORT}:10000"
container_name: bbproxy
image: budibase/proxy
+ environment:
+ - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ - PROXY_RATE_LIMIT_API_PER_SECOND=20
depends_on:
- minio-service
- worker-service
diff --git a/hosting/hosting.properties b/hosting/hosting.properties
index 11dd661bf1..c5638a266f 100644
--- a/hosting/hosting.properties
+++ b/hosting/hosting.properties
@@ -22,4 +22,7 @@ BUDIBASE_ENVIRONMENT=PRODUCTION
# An admin user can be automatically created initially if these are set
BB_ADMIN_USER_EMAIL=
-BB_ADMIN_USER_PASSWORD=
\ No newline at end of file
+BB_ADMIN_USER_PASSWORD=
+
+# A path that is watched for plugin bundles. Any bundles found are imported automatically/
+PLUGINS_DIR=
\ No newline at end of file
diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs
index 9398b7e719..93a07435e5 100644
--- a/hosting/nginx.dev.conf.hbs
+++ b/hosting/nginx.dev.conf.hbs
@@ -15,12 +15,30 @@ http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
+ '"$http_user_agent" "$http_x_forwarded_for" '
+ 'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+ access_log /var/log/nginx/access.log main;
map $http_upgrade $connection_upgrade {
default "upgrade";
}
+ upstream app-service {
+ server {{address}}:4001;
+ keepalive 32;
+ }
+
+ upstream worker-service {
+ server {{address}}:4002;
+ keepalive 32;
+ }
+
+ upstream builder {
+ server {{address}}:3000;
+ keepalive 32;
+ }
+
server {
listen 10000 default_server;
server_name _;
@@ -40,41 +58,118 @@ http {
}
location ~ ^/api/(system|admin|global)/ {
- proxy_pass http://{{ address }}:4002;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://worker-service;
+ }
+
+ location /api/backups/ {
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+ proxy_pass http://app-service;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
}
location /api/ {
proxy_read_timeout 120s;
proxy_connect_timeout 120s;
- proxy_send_timeout 120s;
- proxy_pass http://{{ address }}:4001;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location = / {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /app_ {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /app {
- proxy_pass http://{{ address }}:4001;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://app-service;
}
location /builder {
- proxy_pass http://{{ address }}:3000;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
+ proxy_set_header Connection "";
+
+ proxy_pass http://builder;
rewrite ^/builder(.*)$ /builder/$1 break;
}
location /builder/ {
- proxy_pass http://{{ address }}:3000;
-
proxy_http_version 1.1;
+
+ proxy_set_header Host $host;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+
+ proxy_pass http://builder;
+ }
+
+ location /vite/ {
+ proxy_pass http://builder;
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+ rewrite ^/vite(.*)$ /$1 break;
+ }
+
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://app-service;
}
location / {
diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs
index ac35a2020d..6f0f1b420d 100644
--- a/hosting/nginx.prod.conf.hbs
+++ b/hosting/nginx.prod.conf.hbs
@@ -9,7 +9,11 @@ events {
}
http {
- limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+ # rate limiting
+ limit_req_status 429;
+ limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=${PROXY_RATE_LIMIT_API_PER_SECOND}r/s;
+ limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
+
include /etc/nginx/mime.types;
default_type application/octet-stream;
proxy_set_header Host $host;
@@ -29,7 +33,10 @@ http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
+ '"$http_user_agent" "$http_x_forwarded_for" '
+ 'response_time=$upstream_response_time proxy_host=$proxy_host upstream_addr=$upstream_addr';
+
+ access_log /var/log/nginx/access.log main;
map $http_upgrade $connection_upgrade {
default "upgrade";
@@ -44,11 +51,11 @@ http {
proxy_buffering off;
set $csp_default "default-src 'self'";
- set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
+ set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
- set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
+ set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
@@ -90,24 +97,50 @@ http {
proxy_pass http://$watchtower:8080;
}
{{/if}}
+
location ~ ^/(builder|app_) {
proxy_http_version 1.1;
+
proxy_set_header Connection $connection_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
+
proxy_pass http://$apps:4002;
}
location ~ ^/api/(system|admin|global)/ {
+ proxy_set_header Host $host;
+
proxy_pass http://$worker:4003;
}
location /worker/ {
+ proxy_set_header Host $host;
+
proxy_pass http://$worker:4003;
rewrite ^/worker/(.*)$ /$1 break;
}
+ location /api/backups/ {
+ # calls to export apps are limited
+ limit_req zone=ratelimit burst=20 nodelay;
+
+ # 1800s timeout for app export requests
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_pass http://$apps:4002;
+ }
+
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=20 nodelay;
@@ -122,6 +155,27 @@ http {
proxy_set_header Upgrade $http_upgrade;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
+
+ proxy_pass http://$apps:4002;
+ }
+
+ location /api/webhooks/ {
+ # calls to webhooks are rate limited
+ limit_req zone=webhooks nodelay;
+
+ # Rest of configuration copied from /api/ location above
+ # 120s timeout on API requests
+ proxy_read_timeout 120s;
+ proxy_connect_timeout 120s;
+ proxy_send_timeout 120s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $host;
proxy_pass http://$apps:4002;
}
@@ -131,15 +185,26 @@ http {
rewrite ^/db/(.*)$ /$1 break;
}
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://$apps:4002;
+ }
+
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Host $http_host;
proxy_connect_timeout 300;
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
+
proxy_pass http://$minio:9000;
}
diff --git a/hosting/proxy/10-listen-on-ipv6-by-default.sh b/hosting/proxy/10-listen-on-ipv6-by-default.sh
new file mode 100644
index 0000000000..e2e89388a9
--- /dev/null
+++ b/hosting/proxy/10-listen-on-ipv6-by-default.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+ # ipv6 not available so delete lines from nginx conf
+ if [ -f "$NGINX_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $NGINX_CONF_FILE
+ fi
+ if [ -f "$DEFAULT_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE
+ fi
+ echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+ echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0
diff --git a/hosting/proxy/80-listen-on-ipv6-by-default.sh b/hosting/proxy/80-listen-on-ipv6-by-default.sh
new file mode 100644
index 0000000000..1d62732ea1
--- /dev/null
+++ b/hosting/proxy/80-listen-on-ipv6-by-default.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+ME=$(basename $0)
+NGINX_CONF_FILE="/etc/nginx/nginx.conf"
+DEFAULT_CONF_FILE="/etc/nginx/conf.d/default.conf"
+
+# check if we have ipv6 available
+if [ ! -f "/proc/net/if_inet6" ]; then
+ # ipv6 not available so delete lines from nginx conf
+ if [ -f "$NGINX_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $NGINX_CONF_FILE
+ fi
+ if [ -f "$DEFAULT_CONF_FILE" ]; then
+ sed -i '/listen \[::\]/d' $DEFAULT_CONF_FILE
+ fi
+ echo "$ME: info: ipv6 not available so delete lines from nginx conf"
+else
+ echo "$ME: info: ipv6 is available so no need to delete lines from nginx conf"
+fi
+
+exit 0
\ No newline at end of file
diff --git a/hosting/proxy/Dockerfile b/hosting/proxy/Dockerfile
index a2b17d3333..68e8134750 100644
--- a/hosting/proxy/Dockerfile
+++ b/hosting/proxy/Dockerfile
@@ -1,3 +1,19 @@
FROM nginx:latest
-COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
-COPY error.html /usr/share/nginx/html/error.html
\ No newline at end of file
+
+# nginx.conf
+# use the default nginx behaviour for *.template files which are processed with envsubst
+# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
+ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+
+# IPv6 removal needs to happen after envsubst
+RUN rm -rf /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
+COPY 80-listen-on-ipv6-by-default.sh /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh
+RUN chmod +x /docker-entrypoint.d/80-listen-on-ipv6-by-default.sh
+
+# Error handling
+COPY error.html /usr/share/nginx/html/error.html
+
+# Default environment
+ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
\ No newline at end of file
diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh
index 4c165d12e7..67e1765ca8 100644
--- a/hosting/scripts/build-target-paths.sh
+++ b/hosting/scripts/build-target-paths.sh
@@ -3,15 +3,22 @@
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persisent data & SSH on port 2222
- mkdir -p /home/{search,minio,couch}
- mkdir -p /home/couch/{dbs,views}
- chown -R couchdb:couchdb /home/couch/
+ DATA_DIR=/home
+ WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+ mkdir -p $DATA_DIR/{search,minio,couch}
+ mkdir -p $DATA_DIR/couch/{dbs,views}
+ chown -R couchdb:couchdb $DATA_DIR/couch/
apt update
apt-get install -y openssh-server
- sed -i 's#dir=/opt/couchdb/data/search#dir=/home/search#' /opt/clouseau/clouseau.ini
- sed -i 's#/minio/minio server /minio &#/minio/minio server /home/minio &#' /runner.sh
- sed -i 's#database_dir = ./data#database_dir = /home/couch/dbs#' /opt/couchdb/etc/default.ini
- sed -i 's#view_index_dir = ./data#view_index_dir = /home/couch/views#' /opt/couchdb/etc/default.ini
- sed -i "s/#Port 22/Port 2222/" /etc/ssh/sshd_config
+ echo "root:Docker!" | chpasswd
+ mkdir -p /tmp
+ chmod +x /tmp/ssh_setup.sh \
+ && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+ cp /etc/sshd_config /etc/ssh/sshd_config
/etc/init.d/ssh restart
-fi
+ sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+ sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+else
+ sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+ sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
\ No newline at end of file
diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile
index b5bf17adde..58796f0362 100644
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@@ -19,32 +19,18 @@ ADD packages/worker .
RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
FROM couchdb:3.2.1
-# TARGETARCH can be amd64 or arm e.g. docker build --build-arg TARGETARCH=amd64
-ARG TARGETARCH amd64
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
# e.g. docker build --build-arg TARGETBUILD=aas ....
-ARG TARGETBUILD single
+ARG TARGETBUILD=single
ENV TARGETBUILD $TARGETBUILD
COPY --from=build /app /app
COPY --from=build /worker /worker
-ENV \
- APP_PORT=4001 \
- ARCHITECTURE=amd \
- BUDIBASE_ENVIRONMENT=PRODUCTION \
- CLUSTER_PORT=80 \
- # CUSTOM_DOMAIN=budi001.custom.com \
- DEPLOYMENT_ENVIRONMENT=docker \
- MINIO_URL=http://localhost:9000 \
- POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \
- REDIS_URL=localhost:6379 \
- SELF_HOSTED=1 \
- TARGETBUILD=$TARGETBUILD \
- WORKER_PORT=4002 \
- WORKER_URL=http://localhost:4002 \
- APPS_URL=http://localhost:4001
-
+# ENV CUSTOM_DOMAIN=budi001.custom.com \
+# See runner.sh for Env Vars
# These secret env variables are generated by the runner at startup
# their values can be overriden by the user, they will be written
# to the .env file in the /data directory for use later on
@@ -114,7 +100,10 @@ RUN chmod +x ./healthcheck.sh
ADD hosting/scripts/build-target-paths.sh .
RUN chmod +x ./build-target-paths.sh
+# Script below sets the path for storing data based on $DATA_DIR
# For Azure App Service install SSH & point data locations to /home
+ADD hosting/single/ssh/sshd_config /etc/
+ADD hosting/single/ssh/ssh_setup.sh /tmp
RUN /build-target-paths.sh
# cleanup cache
@@ -122,6 +111,8 @@ RUN yarn cache clean -f
EXPOSE 80
EXPOSE 443
+# Expose port 2222 for SSH on Azure App Service build
+EXPOSE 2222
VOLUME /data
# setup letsencrypt certificate
diff --git a/hosting/single/clouseau/clouseau.ini b/hosting/single/clouseau/clouseau.ini
index 78e43744e5..578a5acafa 100644
--- a/hosting/single/clouseau/clouseau.ini
+++ b/hosting/single/clouseau/clouseau.ini
@@ -7,7 +7,7 @@ name=clouseau@127.0.0.1
cookie=monster
; the path where you would like to store the search index files
-dir=/data/search
+dir=DATA_DIR/search
; the number of search indexes that can be open simultaneously
max_indexes_open=500
diff --git a/hosting/single/couch/local.ini b/hosting/single/couch/local.ini
index 72872a60e1..266c0d4b60 100644
--- a/hosting/single/couch/local.ini
+++ b/hosting/single/couch/local.ini
@@ -1,5 +1,5 @@
; CouchDB Configuration Settings
[couchdb]
-database_dir = /data/couch/dbs
-view_index_dir = /data/couch/views
+database_dir = DATA_DIR/couch/dbs
+view_index_dir = DATA_DIR/couch/views
diff --git a/hosting/single/healthcheck.sh b/hosting/single/healthcheck.sh
index b92cd153a3..592b3e94fa 100644
--- a/hosting/single/healthcheck.sh
+++ b/hosting/single/healthcheck.sh
@@ -3,6 +3,11 @@ healthy=true
if [ -f "/data/.env" ]; then
export $(cat /data/.env | xargs)
+elif [ -f "/home/.env" ]; then
+ export $(cat /home/.env | xargs)
+else
+ echo "No .env file found"
+ healthy=false
fi
if [[ $(curl -Lfk -s -w "%{http_code}\n" http://localhost/ -o /dev/null) -ne 200 ]]; then
diff --git a/hosting/single/nginx/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf
index c0d80a0185..9a5ec91c1f 100644
--- a/hosting/single/nginx/nginx-default-site.conf
+++ b/hosting/single/nginx/nginx-default-site.conf
@@ -43,6 +43,24 @@ server {
rewrite ^/worker/(.*)$ /$1 break;
}
+ location /api/backups/ {
+ # calls to export apps are limited
+ limit_req zone=ratelimit burst=20 nodelay;
+
+ # 1800s timeout for app export requests
+ proxy_read_timeout 1800s;
+ proxy_connect_timeout 1800s;
+ proxy_send_timeout 1800s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+ proxy_pass http://127.0.0.1:4001;
+ }
+
location /api/ {
# calls to the API are rate limited with bursting
limit_req zone=ratelimit burst=20 nodelay;
@@ -66,6 +84,15 @@ server {
rewrite ^/db/(.*)$ /$1 break;
}
+ location /socket/ {
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection 'upgrade';
+ proxy_set_header Host $host;
+ proxy_cache_bypass $http_upgrade;
+ proxy_pass http://127.0.0.1:4001;
+ }
+
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh
index 9abb2fd093..ea825131db 100644
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@@ -1,9 +1,47 @@
#!/bin/bash
-declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
-if [ -f "/data/.env" ]; then
- export $(cat /data/.env | xargs)
+declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
+declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL")
+# Check the env vars set in Dockerfile have come through, AAS seems to drop them
+[[ -z "${APP_PORT}" ]] && export APP_PORT=4001
+[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd
+[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
+[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
+[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
+[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
+[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
+[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
+[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS"
+[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
+[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
+[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
+[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
+[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002
+[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001
+# export CUSTOM_DOMAIN=budi001.custom.com
+
+# Azure App Service customisations
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+ DATA_DIR=/home
+ WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+ /etc/init.d/ssh start
+else
+ DATA_DIR=${DATA_DIR:-/data}
fi
-# first randomise any unset environment variables
+mkdir -p ${DATA_DIR}
+# Mount NFS or GCP Filestore if env vars exist for it
+if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+ echo "Mounting NFS share"
+ apt update && apt install -y nfs-common nfs-kernel-server
+ echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
+ mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
+ echo "Mounting result: $?"
+fi
+
+if [ -f "${DATA_DIR}/.env" ]; then
+ # Read in the .env file and export the variables
+ for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+fi
+# randomise any unset environment variables
for ENV_VAR in "${ENV_VARS[@]}"
do
temp=$(eval "echo \$$ENV_VAR")
@@ -14,21 +52,33 @@ done
if [[ -z "${COUCH_DB_URL}" ]]; then
export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984
fi
-if [ ! -f "/data/.env" ]; then
- touch /data/.env
+if [ ! -f "${DATA_DIR}/.env" ]; then
+ touch ${DATA_DIR}/.env
for ENV_VAR in "${ENV_VARS[@]}"
do
temp=$(eval "echo \$$ENV_VAR")
- echo "$ENV_VAR=$temp" >> /data/.env
+ echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
done
+ for ENV_VAR in "${DOCKER_VARS[@]}"
+ do
+ temp=$(eval "echo \$$ENV_VAR")
+ echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+ done
+ echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env
fi
+# Read in the .env file and export the variables
+for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+ln -s ${DATA_DIR}/.env /app/.env
+ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
-mkdir -p /data/couch/{dbs,views} /home/couch/{dbs,views}
-chown -R couchdb:couchdb /data/couch /home/couch
-redis-server --requirepass $REDIS_PASSWORD &
-/opt/clouseau/bin/clouseau &
-/minio/minio server /data/minio &
+mkdir -p ${DATA_DIR}/couch/{dbs,views}
+mkdir -p ${DATA_DIR}/minio
+mkdir -p ${DATA_DIR}/search
+chown -R couchdb:couchdb ${DATA_DIR}/couch
+redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
/etc/init.d/nginx restart
if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
@@ -37,16 +87,18 @@ if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
chmod +x /etc/cron.d/certificate-renew
# Request the certbot certificate
/app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN}
+ /etc/init.d/nginx restart
fi
-/etc/init.d/nginx restart
pushd app
-pm2 start --name app "yarn run:docker"
+pm2 start -l /dev/stdout --name app "yarn run:docker"
popd
pushd worker
-pm2 start --name worker "yarn run:docker"
+pm2 start -l /dev/stdout --name worker "yarn run:docker"
popd
sleep 10
+echo "curl to couchdb endpoints"
curl -X PUT ${COUCH_DB_URL}/_users
curl -X PUT ${COUCH_DB_URL}/_replicator
+echo "end of runner.sh, sleeping ..."
sleep infinity
diff --git a/hosting/single/ssh/ssh_setup.sh b/hosting/single/ssh/ssh_setup.sh
new file mode 100644
index 0000000000..0af0b6d7ad
--- /dev/null
+++ b/hosting/single/ssh/ssh_setup.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+ssh-keygen -A
+
+#prepare run dir
+if [ ! -d "/var/run/sshd" ]; then
+ mkdir -p /var/run/sshd
+fi
\ No newline at end of file
diff --git a/hosting/single/ssh/sshd_config b/hosting/single/ssh/sshd_config
new file mode 100644
index 0000000000..7eb5df953a
--- /dev/null
+++ b/hosting/single/ssh/sshd_config
@@ -0,0 +1,12 @@
+Port 2222
+ListenAddress 0.0.0.0
+LoginGraceTime 180
+X11Forwarding yes
+Ciphers aes128-cbc,3des-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr
+MACs hmac-sha1,hmac-sha1-96
+StrictModes yes
+SyslogFacility DAEMON
+PasswordAuthentication yes
+PermitEmptyPasswords no
+PermitRootLogin yes
+Subsystem sftp internal-sftp
diff --git a/lerna.json b/lerna.json
index 6394390f18..535393e618 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
{
- "version": "1.2.20",
+ "version": "2.1.40-alpha.0",
"npmClient": "yarn",
"packages": [
"packages/*"
diff --git a/package.json b/package.json
index 4c24e0025b..af513fc8dd 100644
--- a/package.json
+++ b/package.json
@@ -3,7 +3,6 @@
"private": true,
"devDependencies": {
"@rollup/plugin-json": "^4.0.2",
- "@types/mongodb": "3.6.3",
"@typescript-eslint/parser": "4.28.0",
"babel-eslint": "^10.0.3",
"eslint": "^7.28.0",
@@ -13,18 +12,21 @@
"js-yaml": "^4.1.0",
"kill-port": "^1.6.1",
"lerna": "3.14.1",
+ "madge": "^5.0.1",
"prettier": "^2.3.1",
"prettier-plugin-svelte": "^2.3.0",
"rimraf": "^3.0.2",
"rollup-plugin-replace": "^2.2.0",
"svelte": "^3.38.2",
- "typescript": "4.5.5"
+ "typescript": "4.7.3"
},
"scripts": {
"setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
"bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
"build": "lerna run build",
"build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+ "build:sdk": "lerna run build:sdk",
+ "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
"release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
"release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop",
"release:pro": "bash scripts/pro/release.sh",
@@ -45,8 +47,8 @@
"lint:eslint": "eslint packages",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
"lint": "yarn run lint:eslint && yarn run lint:prettier",
- "lint:fix:eslint": "eslint --fix packages",
- "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
+ "lint:fix:eslint": "eslint --fix packages qa-core",
+ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
"lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
"test:e2e": "lerna run cy:test --stream",
"test:e2e:ci": "lerna run cy:ci --stream",
@@ -73,8 +75,8 @@
"env:multi:disable": "lerna run env:multi:disable",
"env:selfhost:enable": "lerna run env:selfhost:enable",
"env:selfhost:disable": "lerna run env:selfhost:disable",
- "env:localdomain:enable": "lerna run env:localdomain:enable",
- "env:localdomain:disable": "lerna run env:localdomain:disable",
+ "env:localdomain:enable": "./scripts/localdomain.sh enable",
+ "env:localdomain:disable": "./scripts/localdomain.sh disable",
"env:account:enable": "lerna run env:account:enable",
"env:account:disable": "lerna run env:account:disable",
"mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable",
diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts
new file mode 100644
index 0000000000..7fac80faa9
--- /dev/null
+++ b/packages/backend-core/__mocks__/aws-sdk.ts
@@ -0,0 +1,15 @@
+const mockS3 = {
+ headBucket: jest.fn().mockReturnThis(),
+ deleteObject: jest.fn().mockReturnThis(),
+ deleteObjects: jest.fn().mockReturnThis(),
+ createBucket: jest.fn().mockReturnThis(),
+ listObjects: jest.fn().mockReturnThis(),
+ promise: jest.fn().mockReturnThis(),
+ catch: jest.fn(),
+}
+
+const AWS = {
+ S3: jest.fn(() => mockS3),
+}
+
+export default AWS
diff --git a/packages/backend-core/accounts.js b/packages/backend-core/accounts.js
deleted file mode 100644
index 47ad03456a..0000000000
--- a/packages/backend-core/accounts.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/cloud/accounts")
diff --git a/packages/backend-core/auth.js b/packages/backend-core/auth.js
deleted file mode 100644
index bbfe3d41dd..0000000000
--- a/packages/backend-core/auth.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/auth")
diff --git a/packages/backend-core/cache.js b/packages/backend-core/cache.js
deleted file mode 100644
index c8bd3c9b6f..0000000000
--- a/packages/backend-core/cache.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const generic = require("./src/cache/generic")
-
-module.exports = {
- user: require("./src/cache/user"),
- app: require("./src/cache/appMetadata"),
- writethrough: require("./src/cache/writethrough"),
- ...generic,
- cache: generic,
-}
diff --git a/packages/backend-core/constants.js b/packages/backend-core/constants.js
deleted file mode 100644
index 4abb7703db..0000000000
--- a/packages/backend-core/constants.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/constants")
diff --git a/packages/backend-core/context.js b/packages/backend-core/context.js
deleted file mode 100644
index aaa0f56f92..0000000000
--- a/packages/backend-core/context.js
+++ /dev/null
@@ -1,22 +0,0 @@
-const {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
-} = require("./src/context")
-
-const identity = require("./src/context/identity")
-
-module.exports = {
- getAppDB,
- getDevAppDB,
- getProdAppDB,
- getAppId,
- updateAppId,
- doInAppContext,
- doInTenant,
- identity,
-}
diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js
deleted file mode 100644
index 0d2869d9f1..0000000000
--- a/packages/backend-core/db.js
+++ /dev/null
@@ -1,7 +0,0 @@
-module.exports = {
- ...require("./src/db/utils"),
- ...require("./src/db/constants"),
- ...require("./src/db"),
- ...require("./src/db/views"),
- ...require("./src/db/pouch"),
-}
diff --git a/packages/backend-core/deprovision.js b/packages/backend-core/deprovision.js
deleted file mode 100644
index 672da214ff..0000000000
--- a/packages/backend-core/deprovision.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/context/deprovision")
diff --git a/packages/backend-core/encryption.js b/packages/backend-core/encryption.js
deleted file mode 100644
index 4ccb6e3a99..0000000000
--- a/packages/backend-core/encryption.js
+++ /dev/null
@@ -1 +0,0 @@
-module.exports = require("./src/security/encryption")
diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts
new file mode 100644
index 0000000000..d0e5d3d4e7
--- /dev/null
+++ b/packages/backend-core/jest.config.ts
@@ -0,0 +1,21 @@
+import { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+ preset: "ts-jest",
+ testEnvironment: "node",
+ setupFiles: ["./tests/jestSetup.ts"],
+ collectCoverageFrom: ["src/**/*.{js,ts}"],
+ coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+ // use sources when not in CI
+ config.moduleNameMapper = {
+ "@budibase/types": "