diff --git a/.eslintrc.json b/.eslintrc.json index 75584b8163..f6f03c6523 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,6 +19,7 @@ "bundle.js" ], "extends": ["eslint:recommended"], + "plugins": ["import", "eslint-plugin-local-rules"], "overrides": [ { "files": ["**/*.svelte"], @@ -30,7 +31,6 @@ "sourceType": "module", "allowImportExportEverywhere": true } - }, { "files": ["**/*.ts"], @@ -42,13 +42,22 @@ "no-case-declarations": "off", "no-useless-escape": "off", "no-undef": "off", - "no-prototype-builtins": "off" + "no-prototype-builtins": "off", + "local-rules/no-budibase-imports": "error" } } ], "rules": { "no-self-assign": "off", - "no-unused-vars": ["error", { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }] + "no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_" + } + ], + "import/no-relative-packages": "error" }, "globals": { "GeolocationPositionError": true diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 77867c8617..6e04ca6f67 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -12,6 +12,13 @@ on: - master pull_request: workflow_dispatch: + workflow_call: + inputs: + run_as_oss: + type: boolean + required: false + description: Force running checks as if it was an OSS contributor + default: false env: BRANCH: ${{ github.event.pull_request.head.ref }} @@ -19,50 +26,41 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }} + IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - run: yarn lint build: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile # Run build all the projects @@ -81,24 +79,18 @@ jobs: test-libraries: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -116,24 +108,18 @@ jobs: test-worker: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test worker run: | @@ -152,24 +138,18 @@ jobs: test-server: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test server run: | @@ -200,7 +180,7 @@ jobs: uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -213,24 +193,23 @@ jobs: integration-test: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Build packages run: yarn build --scope @budibase/server --scope @budibase/worker + - name: Build backend-core for OSS contributor (required for pro) + if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }} + run: yarn build --scope @budibase/backend-core - name: Run tests run: | cd qa-core diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml new file mode 100644 index 0000000000..398f07a130 --- /dev/null +++ b/.github/workflows/check-oss-contributor.yml @@ -0,0 +1,35 @@ +name: OSS contributor checks +on: + workflow_dispatch: + schedule: + - cron: "0 8,16 * * 1-5" # on weekdays at 8am and 4pm + +jobs: + run-checks: + name: Publish server and worker docker images + uses: ./.github/workflows/budibase_ci.yml + with: + run_as_oss: true + secrets: inherit + + notify-error: + needs: ["run-checks"] + if: ${{ failure() }} + name: Notify error + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set commit SHA + id: set_sha + run: echo "::set-output name=sha::$(git rev-parse --short ${{ github.sha }})" + + - name: Notify error + uses: tsickert/discord-webhook@v5.3.0 + with: + webhook-url: ${{ secrets.OSS_CHECKS_WEBHOOK_URL }} + embed-title: 🚨 OSS checks failed in master + embed-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + embed-description: | + Git sha: `${{ steps.set_sha.outputs.sha }}` diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml deleted file mode 100644 index 389b10f7d3..0000000000 --- a/.github/workflows/deploy-cloud.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Budibase Deploy Production - -on: - workflow_dispatch: - inputs: - version: - description: Budibase release version. For example - 1.0.0 - required: false - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - if [ -z "${{ github.event.inputs.version }}" ]; then - release_version=$(cat lerna.json | jq -r '.version') - else - release_version=${{ github.event.inputs.version }} - fi - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-prod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index f1fb12c087..c70f2fff20 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -7,6 +7,7 @@ on: jobs: release: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml deleted file mode 100644 index 2edb470405..0000000000 --- a/.github/workflows/release-master.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Budibase Release -concurrency: - group: release - cancel-in-progress: false - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - # Exclude all pre-releases - - "!*[0-9]+.[0-9]+.[0-9]+-*" - -env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - -jobs: - release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - cache: yarn - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default - git config --global user.name "Budibase Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Docker login - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - - name: Build worker docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/worker/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/worker - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build server docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/server/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/apps - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build proxy docker - uses: docker/build-push-action@v5 - with: - context: ./hosting/proxy - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./hosting/proxy/Dockerfile - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/proxy - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: peter-evans/repository-dispatch@v2 - with: - repository: budibase/budibase-deploys - event-type: budicloud-qa-deploy - token: ${{ secrets.GH_ACCESS_TOKEN }} - client-payload: |- - { - "VERSION": "${{ github.ref_name }}", - "REF_NAME": "${{ github.ref_name}}" - } diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml deleted file mode 100644 index d2689a0ea0..0000000000 --- a/.github/workflows/release-selfhost.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Budibase Release Selfhost - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Use Node.js 18.x - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - name: Tag and release Docker images (Self Host) - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - release_tag=${{ env.RELEASE_VERSION }} - - # Pull apps and worker images - docker pull budibase/apps:$release_tag - docker pull budibase/worker:$release_tag - docker pull budibase/proxy:$release_tag - - # Tag apps and worker images - docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG - docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG - docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG - - # Push images - docker push budibase/apps:$SELFHOST_TAG - docker push budibase/worker:$SELFHOST_TAG - docker push budibase/proxy:$SELFHOST_TAG - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - SELFHOST_TAG: latest - - - name: Bootstrap and build (CLI) - run: | - yarn - yarn build - - - name: Build OpenAPI spec - run: | - pushd packages/server - yarn - yarn specs - popd - - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Perform Github Release - uses: softprops/action-gh-release@v1 - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - generate_release_notes: true - files: | - packages/cli/build/cli-win.exe - packages/cli/build/cli-linux - packages/cli/build/cli-macos - packages/server/specs/openapi.yaml - packages/server/specs/openapi.json - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 - with: - webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml deleted file mode 100644 index 16b1da186a..0000000000 --- a/.github/workflows/release-singleimage.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Deploy Budibase Single Container Image to DockerHub - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 30000 - swap-size-mb: 1024 - remove-android: "true" - remove-dotnet: "true" - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - name: "Checkout" - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Run Yarn - run: yarn - - name: Update versions - run: ./scripts/updateVersions.sh - - name: Run Yarn Build - run: yarn build - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: | - TARGETBUILD=aas - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index 78c07a037c..13d59d1019 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -45,8 +45,8 @@ jobs: BUMP_TYPE=${BUMP_TYPE_INPUT:-"patch"} ./versionCommit.sh $BUMP_TYPE - - new_version=$(./getCurrentVersion.sh) + cd .. + new_version=$(./scripts/getCurrentVersion.sh) echo "version=$new_version" >> $GITHUB_OUTPUT trigger-release: diff --git a/.prettierignore b/.prettierignore index 64607d74ab..ce7617224b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,13 +1,11 @@ node_modules dist -*.spec.js -packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte packages/server/builder packages/server/coverage -packages/worker/coverage -packages/backend-core/coverage packages/server/client packages/server/src/definitions/openapi.ts +packages/worker/coverage +packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk packages/pro/coverage \ No newline at end of file diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml index 41af2624bf..f98ecc139d 100644 --- a/charts/budibase/templates/minio-service-deployment.yaml +++ b/charts/budibase/templates/minio-service-deployment.yaml @@ -46,11 +46,9 @@ spec: image: minio/minio imagePullPolicy: "" livenessProbe: - exec: - command: - - curl - - -f - - http://localhost:9000/minio/health/live + httpGet: + path: /minio/health/live + port: 9000 failureThreshold: 3 periodSeconds: 30 timeoutSeconds: 20 diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js new file mode 100644 index 0000000000..af02599c90 --- /dev/null +++ b/eslint-local-rules/index.js @@ -0,0 +1,21 @@ +module.exports = { + "no-budibase-imports": { + create: function (context) { + return { + ImportDeclaration(node) { + const importPath = node.source.value + + if ( + /^@budibase\/[^/]+\/.*$/.test(importPath) && + importPath !== "@budibase/backend-core/tests" + ) { + context.report({ + node, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`, + }) + } + }, + } + }, + }, +} diff --git a/hosting/couchdb/build-target-paths.sh b/hosting/couchdb/build-target-paths.sh index 67e1765ca8..34227011f4 100644 --- a/hosting/couchdb/build-target-paths.sh +++ b/hosting/couchdb/build-target-paths.sh @@ -2,8 +2,8 @@ echo ${TARGETBUILD} > /buildtarget.txt if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persisent data & SSH on port 2222 - DATA_DIR=/home + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true mkdir -p $DATA_DIR/{search,minio,couch} mkdir -p $DATA_DIR/couch/{dbs,views} diff --git a/hosting/letsencrypt/nginx-ssl.conf b/hosting/letsencrypt/nginx-ssl.conf index 50c5e0198a..b3f51e5cc5 100644 --- a/hosting/letsencrypt/nginx-ssl.conf +++ b/hosting/letsencrypt/nginx-ssl.conf @@ -2,16 +2,18 @@ server { listen 443 ssl default_server; listen [::]:443 ssl default_server; server_name _; - ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem; - include /etc/letsencrypt/options-ssl-nginx.conf; - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - + error_log /dev/stderr warn; + access_log /dev/stdout main; client_max_body_size 1000m; ignore_invalid_headers off; proxy_buffering off; # port_in_redirect off; + ssl_certificate /etc/letsencrypt/live/CUSTOM_DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + location ^~ /.well-known/acme-challenge/ { default_type "text/plain"; root /var/www/html; @@ -47,6 +49,24 @@ server { rewrite ^/worker/(.*)$ /$1 break; } + location /api/backups/ { + # calls to export apps are limited + limit_req zone=ratelimit burst=20 nodelay; + + # 1800s timeout for app export requests + proxy_read_timeout 1800s; + proxy_connect_timeout 1800s; + proxy_send_timeout 1800s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://127.0.0.1:4001; + } + location /api/ { # calls to the API are rate limited with bursting limit_req zone=ratelimit burst=20 nodelay; @@ -70,18 +90,49 @@ server { rewrite ^/db/(.*)$ /$1 break; } + location /socket/ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_pass http://127.0.0.1:4001; + } + location / { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; proxy_connect_timeout 300; proxy_http_version 1.1; proxy_set_header Connection ""; chunked_transfer_encoding off; + proxy_pass http://127.0.0.1:9000; } + location /files/signed/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # IMPORTANT: Signed urls will inspect the host header of the request. + # Normally a signed url will need to be generated with a specified client host in mind. + # To support dynamic hosts, e.g. some unknown self-hosted installation url, + # use a predefined host header. The host 'minio-service' is also used at the time of url signing. + proxy_set_header Host minio-service; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://127.0.0.1:9000; + rewrite ^/files/signed/(.*)$ /$1 break; + } + client_header_timeout 60; client_body_timeout 60; keepalive_timeout 60; diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh index 67e1765ca8..34227011f4 100644 --- a/hosting/scripts/build-target-paths.sh +++ b/hosting/scripts/build-target-paths.sh @@ -2,8 +2,8 @@ echo ${TARGETBUILD} > /buildtarget.txt if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persisent data & SSH on port 2222 - DATA_DIR=/home + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true mkdir -p $DATA_DIR/{search,minio,couch} mkdir -p $DATA_DIR/couch/{dbs,views} diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..87201c95c0 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -22,7 +22,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME # Azure App Service customisations if [[ "${TARGETBUILD}" = "aas" ]]; then - DATA_DIR=/home + DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true /etc/init.d/ssh start else diff --git a/lerna.json b/lerna.json index 611cf7d32b..9b22d286b5 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.0", + "version": "2.13.12", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 8a27cde104..2978483448 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,17 @@ "name": "root", "private": true, "devDependencies": { + "@babel/core": "^7.22.5", + "@babel/eslint-parser": "^7.22.5", + "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-local-rules": "^2.0.0", + "eslint-plugin-svelte": "^2.32.2", "husky": "^8.0.3", "kill-port": "^1.6.1", "lerna": "7.1.1", @@ -17,12 +23,8 @@ "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "svelte": "3.49.0", - "typescript": "5.2.2", - "@babel/core": "^7.22.5", - "@babel/eslint-parser": "^7.22.5", - "@babel/preset-env": "^7.22.5", - "eslint-plugin-svelte": "^2.32.2", - "svelte-eslint-parser": "^0.32.0" + "svelte-eslint-parser": "^0.32.0", + "typescript": "5.2.2" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index bd3efc20db..d442511fb8 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -19,7 +19,7 @@ async function populateFromDB(appId: string) { return doWithDB( appId, (db: Database) => { - return db.get(DocumentType.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7cd5d6227f..7a2be5a0f0 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -1,6 +1,6 @@ -const BaseCache = require("./base") +import BaseCache from "./base" -const GENERIC = new BaseCache.default() +const GENERIC = new BaseCache() export enum CacheKey { CHECKLIST = "checklist", @@ -19,6 +19,7 @@ export enum TTL { } function performExport(funcName: string) { + // @ts-ignore return (...args: any) => GENERIC[funcName](...args) } diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts index 58928c271a..4fa986e4e2 100644 --- a/packages/backend-core/src/cache/index.ts +++ b/packages/backend-core/src/cache/index.ts @@ -2,4 +2,6 @@ export * as generic from "./generic" export * as user from "./user" export * as app from "./appMetadata" export * as writethrough from "./writethrough" +export * as invite from "./invite" +export * as passwordReset from "./passwordReset" export * from "./generic" diff --git a/packages/backend-core/src/cache/invite.ts b/packages/backend-core/src/cache/invite.ts new file mode 100644 index 0000000000..e43ebc4aa8 --- /dev/null +++ b/packages/backend-core/src/cache/invite.ts @@ -0,0 +1,86 @@ +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" +import env from "../environment" +import { getTenantId } from "../context" +import * as redis from "../redis/init" + +const TTL_SECONDS = Duration.fromDays(7).toSeconds() + +interface Invite { + email: string + info: any +} + +interface InviteWithCode extends Invite { + code: string +} + +/** + * Given an invite code and invite body, allow the update an existing/valid invite in redis + * @param code The invite code for an invite in redis + * @param value The body of the updated user invitation + */ +export async function updateCode(code: string, value: Invite) { + const client = await redis.getInviteClient() + await client.store(code, value, TTL_SECONDS) +} + +/** + * Generates an invitation code and writes it to redis - which can later be checked for user creation. + * @param email the email address which the code is being sent to (for use later). + * @param info Information to be carried along with the invitation. + * @return returns the code that was stored to redis. + */ +export async function createCode(email: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getInviteClient() + await client.store(code, { email, info }, TTL_SECONDS) + return code +} + +/** + * Checks that the provided invite code is valid - will return the email address of user that was invited. + * @param code the invite code that was provided as part of the link. + * @return If the code is valid then an email address will be returned. + */ +export async function getCode(code: string): Promise { + const client = await redis.getInviteClient() + const value = (await client.get(code)) as Invite | undefined + if (!value) { + throw "Invitation is not valid or has expired, please request a new one." + } + return value +} + +export async function deleteCode(code: string) { + const client = await redis.getInviteClient() + await client.delete(code) +} + +/** + Get all currently available user invitations for the current tenant. + **/ +export async function getInviteCodes(): Promise { + const client = await redis.getInviteClient() + const invites: { key: string; value: Invite }[] = await client.scan() + + const results: InviteWithCode[] = invites.map(invite => { + return { + ...invite.value, + code: invite.key, + } + }) + if (!env.MULTI_TENANCY) { + return results + } + const tenantId = getTenantId() + return results.filter(invite => tenantId === invite.info.tenantId) +} + +export async function getExistingInvites( + emails: string[] +): Promise { + return (await getInviteCodes()).filter(invite => + emails.includes(invite.email) + ) +} diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts new file mode 100644 index 0000000000..7f5a93f149 --- /dev/null +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -0,0 +1,38 @@ +import * as redis from "../redis/init" +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" + +const TTL_SECONDS = Duration.fromHours(1).toSeconds() + +interface PasswordReset { + userId: string + info: any +} + +/** + * Given a user ID this will store a code (that is returned) for an hour in redis. + * The user can then return this code for resetting their password (through their reset link). + * @param userId the ID of the user which is to be reset. + * @param info Info about the user/the reset process. + * @return returns the code that was stored to redis. + */ +export async function createCode(userId: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getPasswordResetClient() + await client.store(code, { userId, info }, TTL_SECONDS) + return code +} + +/** + * Given a reset code this will lookup to redis, check if the code is valid. + * @param code The code provided via the email link. + * @return returns the user ID if it is found + */ +export async function getCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + const value = (await client.get(code)) as PasswordReset | undefined + if (!value) { + throw "Provided information is not valid, cannot reset password - please try again." + } + return value +} diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index b33b4835a9..bb944556af 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -28,7 +28,7 @@ export enum ViewName { APP_BACKUP_BY_TRIGGER = "by_trigger", } -export const DeprecatedViews = { +export const DeprecatedViews: Record = { [ViewName.USER_BY_EMAIL]: [ // removed due to inaccuracy in view doc filter logic "by_email", diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index d29b6935a8..a59f5c6503 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -4,7 +4,7 @@ import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() - static run(context: ContextMap, func: any) { + static run(context: ContextMap, func: () => T) { return Context.storage.run(context, () => func()) } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 609c18abb5..d2259cfcab 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap { return context } -async function newContext(updates: ContextMap, task: any) { +async function newContext(updates: ContextMap, task: () => T) { // see if there already is a context setup let context: ContextMap = updateContext(updates) return Context.run(context, task) } -export async function doInAutomationContext(params: { +export async function doInAutomationContext(params: { appId: string automationId: string - task: any -}): Promise { + task: () => T +}): Promise { const tenantId = getTenantIDFromAppID(params.appId) return newContext( { @@ -144,10 +144,10 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext( +export async function doInAppContext( appId: string | null, - task: any -): Promise { + task: () => T +): Promise { if (!appId && !env.isTest()) { throw new Error("appId is required") } @@ -165,10 +165,10 @@ export async function doInAppContext( return newContext(updates, task) } -export async function doInIdentityContext( +export async function doInIdentityContext( identity: IdentityContext, - task: any -): Promise { + task: () => T +): Promise { if (!identity) { throw new Error("identity is required") } @@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database { */ export function getAppDB(opts?: any): Database { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve app DB - no app ID.") + } return getDB(appId, opts) } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 29ca4123f5..8588a7157a 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -10,6 +10,7 @@ import { DatabaseDeleteIndexOpts, Document, isDocument, + RowResponse, } from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" @@ -48,10 +49,7 @@ export class DatabaseImpl implements Database { private readonly couchInfo = getCouchInfo() - constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { - if (dbName == null) { - throw new Error("Database name cannot be undefined.") - } + constructor(dbName: string, opts?: DatabaseOpts, connection?: string) { this.name = dbName this.pouchOpts = opts || {} if (connection) { @@ -112,7 +110,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -120,6 +118,35 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const rowUnavailable = (row: RowResponse) => { + // row is deleted - key lookup can return this + if (row.doc == null || ("deleted" in row.value && row.value.deleted)) { + return true + } + return row.error === "not_found" + } + + const rows = response.rows.filter(row => !rowUnavailable(row)) + const someMissing = rows.length !== response.rows.length + // some were filtered out - means some missing + if (!opts?.allowMissing && someMissing) { + const missing = response.rows.filter(row => rowUnavailable(row)) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string @@ -175,12 +202,14 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: DatabaseQueryOpts): Promise> { + async allDocs( + params: DatabaseQueryOpts + ): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query( + async query( viewName: string, params: DatabaseQueryOpts ): Promise> { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 9aae64b892..3e69d49f0e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,10 +1,7 @@ -import env from "../environment" import { directCouchQuery, DatabaseImpl } from "./couch" -import { CouchFindOptions, Database } from "@budibase/types" +import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" -const dbList = new Set() - -export function getDB(dbName?: string, opts?: any): Database { +export function getDB(dbName: string, opts?: DatabaseOpts): Database { return new DatabaseImpl(dbName, opts) } @@ -14,7 +11,7 @@ export function getDB(dbName?: string, opts?: any): Database { export async function doWithDB( dbName: string, cb: (db: Database) => Promise, - opts = {} + opts?: DatabaseOpts ) { const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all @@ -22,13 +19,6 @@ export async function doWithDB( return await cb(db) } -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - return [...dbList] -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js index 0d257f7ed7..e03c9a5b0e 100644 --- a/packages/backend-core/src/db/tests/index.spec.js +++ b/packages/backend-core/src/db/tests/index.spec.js @@ -5,7 +5,6 @@ const { getDB } = require("../db") describe("db", () => { describe("getDB", () => { it("returns a db", async () => { - const dbName = structures.db.id() const db = getDB(dbName) expect(db).toBeDefined() diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0980ad217..5d9c5b74d3 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -7,12 +7,19 @@ import { } from "../constants" import { getGlobalDB } from "../context" import { doWithDB } from "./" -import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types" +import { + AllDocsResponse, + Database, + DatabaseQueryOpts, + Document, + DesignDocument, + DBView, +} from "@budibase/types" import env from "../environment" const DESIGN_DB = "_design/database" -function DesignDoc() { +function DesignDoc(): DesignDocument { return { _id: DESIGN_DB, // view collation information, read before writing any complex views: @@ -21,20 +28,14 @@ function DesignDoc() { } } -interface DesignDocument { - views: any -} - async function removeDeprecated(db: Database, viewName: ViewName) { - // @ts-ignore if (!DeprecatedViews[viewName]) { return } try { const designDoc = await db.get(DESIGN_DB) - // @ts-ignore for (let deprecatedNames of DeprecatedViews[viewName]) { - delete designDoc.views[deprecatedNames] + delete designDoc.views?.[deprecatedNames] } await db.put(designDoc) } catch (err) { @@ -43,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } export async function createView( - db: any, + db: Database, viewJs: string, viewName: string ): Promise { let designDoc try { - designDoc = (await db.get(DESIGN_DB)) as DesignDocument + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() } - const view = { + const view: DBView = { map: viewJs, } designDoc.views = { @@ -109,7 +110,7 @@ export interface QueryViewOptions { arrayResponse?: boolean } -export async function queryViewRaw( +export async function queryViewRaw( viewName: ViewName, params: DatabaseQueryOpts, db: Database, @@ -137,18 +138,16 @@ export async function queryViewRaw( } } -export const queryView = async ( +export const queryView = async ( viewName: ViewName, params: DatabaseQueryOpts, db: Database, createFunc: any, opts?: QueryViewOptions -): Promise => { +): Promise => { const response = await queryViewRaw(viewName, params, db, createFunc, opts) const rows = response.rows - const docs = rows.map((row: any) => - params.include_docs ? row.doc : row.value - ) + const docs = rows.map(row => (params.include_docs ? row.doc! : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -198,11 +197,11 @@ export const createPlatformUserView = async () => { await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE) } -export const queryPlatformView = async ( +export const queryPlatformView = async ( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions -): Promise => { +): Promise => { const CreateFuncByName: any = { [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView, [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, @@ -220,7 +219,7 @@ const CreateFuncByName: any = { [ViewName.USER_BY_APP]: createUserAppView, } -export const queryGlobalView = async ( +export const queryGlobalView = async ( viewName: ViewName, params: DatabaseQueryOpts, db?: Database, @@ -231,10 +230,10 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db!, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } -export async function queryGlobalViewRaw( +export async function queryGlobalViewRaw( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..2cfd517941 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,7 +30,9 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" +export * as docIds from "./docIds" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies @@ -49,6 +51,7 @@ export * from "./constants" // expose package init function import * as db from "./db" + export const init = (opts: any = {}) => { db.init(opts.db) } diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..0657437a3b 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,11 +2,17 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" -const CLEANUP_PERIOD_MS = 60 * 1000 +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() +// cleanup the queue every 60 seconds +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -20,8 +26,15 @@ export function createQueue( jobQueue: JobQueue, opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { - const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const redisOpts = getRedisOptions() + const queueConfig: QueueOptions = { + redis: redisOpts, + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index 55ffe3dd12..f3bcee3209 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -7,15 +7,19 @@ let userClient: Client, cacheClient: Client, writethroughClient: Client, lockClient: Client, - socketClient: Client + socketClient: Client, + inviteClient: Client, + passwordResetClient: Client -async function init() { +export async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init() + inviteClient = await new Client(utils.Databases.INVITATIONS).init() + passwordResetClient = await new Client(utils.Databases.PW_RESETS).init() socketClient = await new Client( utils.Databases.SOCKET_IO, utils.SelectableDatabase.SOCKET_IO @@ -29,6 +33,8 @@ export async function shutdown() { if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() if (lockClient) await lockClient.finish() + if (inviteClient) await inviteClient.finish() + if (passwordResetClient) await passwordResetClient.finish() if (socketClient) await socketClient.finish() } @@ -84,3 +90,17 @@ export async function getSocketClient() { } return socketClient } + +export async function getInviteClient() { + if (!inviteClient) { + await init() + } + return inviteClient +} + +export async function getPasswordResetClient() { + if (!passwordResetClient) { + await init() + } + return passwordResetClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index d1e2d8989e..701e262091 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -16,6 +16,7 @@ import { getRedisOptions, SEPARATOR, SelectableDatabase, + getRedisConnectionDetails, } from "./utils" import * as timers from "../timers" @@ -27,7 +28,6 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false let CLIENTS: { [key: number]: any } = {} -0 let CONNECTED = false // mock redis always connected @@ -91,12 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) { if (client) { client.disconnect() } - const { redisProtocolUrl, opts, host, port } = getRedisOptions() + const { host, port } = getRedisConnectionDetails() + const opts = getRedisOptions() if (CLUSTERED) { client = new RedisCore.Cluster([{ host, port }], opts) - } else if (redisProtocolUrl) { - client = new RedisCore(redisProtocolUrl) } else { client = new RedisCore(opts) } diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 7fe61a409e..266f1fe989 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -3,6 +3,7 @@ import { getLockClient } from "./init" import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" import env from "../environment" +import { logWarn } from "../logging" async function getClient( type: LockType, @@ -116,7 +117,7 @@ export async function doWithLock( const result = await task() return { executed: true, result } } catch (e: any) { - console.warn("lock error") + logWarn(`lock type: ${opts.type} error`, e) // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { @@ -124,11 +125,9 @@ export async function doWithLock( // due to retry count (0) exceeded return { executed: false } } else { - console.error(e) throw e } } else { - console.error(e) throw e } } finally { diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 34b7275a2b..4d8b1bb9a4 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -1,4 +1,5 @@ import env from "../environment" +import * as Redis from "ioredis" const SLOT_REFRESH_MS = 2000 const CONNECT_TIMEOUT_MS = 10000 @@ -42,7 +43,7 @@ export enum Databases { export enum SelectableDatabase { DEFAULT = 0, SOCKET_IO = 1, - UNUSED_1 = 2, + RATE_LIMITING = 2, UNUSED_2 = 3, UNUSED_3 = 4, UNUSED_4 = 5, @@ -58,7 +59,7 @@ export enum SelectableDatabase { UNUSED_14 = 15, } -export function getRedisOptions() { +export function getRedisConnectionDetails() { let password = env.REDIS_PASSWORD let url: string[] | string = env.REDIS_URL.split("//") // get rid of the protocol @@ -74,28 +75,36 @@ export function getRedisOptions() { } const [host, port] = url.split(":") - let redisProtocolUrl - - // fully qualified redis URL - if (/rediss?:\/\//.test(env.REDIS_URL)) { - redisProtocolUrl = env.REDIS_URL + const portNumber = parseInt(port) + return { + host, + password, + // assume default port for redis if invalid found + port: isNaN(portNumber) ? 6379 : portNumber, } +} - const opts: any = { +export function getRedisOptions() { + const { host, password, port } = getRedisConnectionDetails() + let redisOpts: Redis.RedisOptions = { connectTimeout: CONNECT_TIMEOUT_MS, + port: port, + host, + password, } + let opts: Redis.ClusterOptions | Redis.RedisOptions = redisOpts if (env.REDIS_CLUSTERED) { - opts.redisOptions = {} - opts.redisOptions.tls = {} - opts.redisOptions.password = password - opts.slotsRefreshTimeout = SLOT_REFRESH_MS - opts.dnsLookup = (address: string, callback: any) => callback(null, address) - } else { - opts.host = host - opts.port = port - opts.password = password + opts = { + connectTimeout: CONNECT_TIMEOUT_MS, + redisOptions: { + ...redisOpts, + tls: {}, + }, + slotsRefreshTimeout: SLOT_REFRESH_MS, + dnsLookup: (address: string, callback: any) => callback(null, address), + } as Redis.ClusterOptions } - return { opts, host, port: parseInt(port), redisProtocolUrl } + return opts } export function addDbPrefix(db: string, key: string) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index c071064713..bd85097bbd 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -303,7 +303,7 @@ export class UserDB { static async bulkCreate( newUsersRequested: User[], - groups: string[] + groups?: string[] ): Promise { const tenantId = getTenantId() @@ -328,7 +328,7 @@ export class UserDB { }) continue } - newUser.userGroups = groups + newUser.userGroups = groups || [] newUsers.push(newUser) if (isCreator(newUser)) { newCreators.push(newUser) @@ -413,15 +413,13 @@ export class UserDB { } // Get users and delete - const allDocsResponse: AllDocsResponse = await db.allDocs({ + const allDocsResponse = await db.allDocs({ include_docs: true, keys: userIds, }) - const usersToDelete: User[] = allDocsResponse.rows.map( - (user: RowResponse) => { - return user.doc - } - ) + const usersToDelete = allDocsResponse.rows.map(user => { + return user.doc! + }) // Delete from DB const toDelete = usersToDelete.map(user => ({ diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts index 17d0e91d88..355be74dab 100644 --- a/packages/backend-core/src/users/lookup.ts +++ b/packages/backend-core/src/users/lookup.ts @@ -6,6 +6,7 @@ import { } from "@budibase/types" import * as dbUtils from "../db" import { ViewName } from "../constants" +import { getExistingInvites } from "../cache/invite" /** * Apply a system-wide search on emails: @@ -26,6 +27,9 @@ export async function searchExistingEmails(emails: string[]) { const existingAccounts = await getExistingAccounts(emails) matchedEmails.push(...existingAccounts.map(account => account.email)) + const invitedEmails = await getExistingInvites(emails) + matchedEmails.push(...invitedEmails.map(invite => invite.email)) + return [...new Set(matchedEmails.map(email => email.toLowerCase()))] } diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 6dc8750b62..9f4a41f6df 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async ( include_docs: true, }) params.startkey = opts && opts.startkey ? opts.startkey : params.startkey - let response = await queryGlobalView(ViewName.USER_BY_APP, params) + let response = await queryGlobalView(ViewName.USER_BY_APP, params) if (!response) { response = [] diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..3c7ef23b11 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,52 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + toSeconds: () => { + return Duration.convert(from, DurationType.SECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index 5a0ac4f283..7b411e801c 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -188,4 +188,17 @@ describe("utils", () => { expectResult(false) }) }) + + describe("hasCircularStructure", () => { + it("should detect a circular structure", () => { + const a: any = { b: "b" } + const b = { a } + a.b = b + expect(utils.hasCircularStructure(b)).toBe(true) + }) + + it("should allow none circular structures", () => { + expect(utils.hasCircularStructure({ a: "b" })).toBe(false) + }) + }) }) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b92471a7a4..1c1ca8473b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -237,3 +237,17 @@ export function timeout(timeMs: number) { export function isAudited(event: Event) { return !!AuditedEventFriendlyName[event] } + +export function hasCircularStructure(json: any) { + if (typeof json !== "object") { + return false + } + try { + JSON.stringify(json) + } catch (err) { + if (err instanceof Error && err?.message.includes("circular structure")) { + return true + } + } + return false +} diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js index ae7109344a..f08c435b95 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -1,5 +1,5 @@ -const _ = require('lodash/fp') -const {structures} = require("../../../tests") +const _ = require("lodash/fp") +const { structures } = require("../../../tests") jest.mock("../../../src/context") jest.mock("../../../src/db") @@ -7,10 +7,9 @@ jest.mock("../../../src/db") const context = require("../../../src/context") const db = require("../../../src/db") -const {getCreatorCount} = require('../../../src/users/users') +const { getCreatorCount } = require("../../../src/users/users") describe("Users", () => { - let getGlobalDBMock let getGlobalUserParamsMock let paginationMock @@ -26,26 +25,26 @@ describe("Users", () => { it("Retrieves the number of creators", async () => { const getUsers = (offset, limit, creators = false) => { const range = _.range(offset, limit) - const opts = creators ? {builder: {global: true}} : undefined + const opts = creators ? { builder: { global: true } } : undefined return range.map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) getGlobalDBMock.mockImplementation(() => ({ - name : "fake-db", + name: "fake-db", allDocs: () => ({ - rows: [...page1Data, ...page2Data] - }) + rows: [...page1Data, ...page2Data], + }), })) paginationMock.mockImplementationOnce(() => ({ data: page1Data, hasNextPage: true, - nextPage: "1" + nextPage: "1", })) paginationMock.mockImplementation(() => ({ data: page2Data, hasNextPage: false, - nextPage: undefined + nextPage: undefined, })) const creatorsCount = await getCreatorCount() expect(creatorsCount).toBe(4) diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 66d23696e0..68ee29686c 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -12,7 +12,7 @@ import { generator } from "./generator" import { tenant } from "." export const newEmail = () => { - return `${uuid()}@test.com` + return `${uuid()}@example.com` } export const user = (userProps?: Partial>): User => { diff --git a/packages/bbui/src/Form/Checkbox.svelte b/packages/bbui/src/Form/Checkbox.svelte index 6aa88f6dee..6e6cf0d8e8 100644 --- a/packages/bbui/src/Form/Checkbox.svelte +++ b/packages/bbui/src/Form/Checkbox.svelte @@ -10,6 +10,7 @@ export let disabled = false export let error = null export let size = "M" + export let helpText = null const dispatch = createEventDispatcher() const onChange = e => { @@ -18,6 +19,6 @@ } - + diff --git a/packages/bbui/src/Form/Combobox.svelte b/packages/bbui/src/Form/Combobox.svelte index 343af559cb..44854d949e 100644 --- a/packages/bbui/src/Form/Combobox.svelte +++ b/packages/bbui/src/Form/Combobox.svelte @@ -11,6 +11,7 @@ export let error = null export let placeholder = "Choose an option or type" export let options = [] + export let helpText = null export let getOptionLabel = option => extractProperty(option, "label") export let getOptionValue = option => extractProperty(option, "value") @@ -27,7 +28,7 @@ } - + diff --git a/packages/bbui/src/Form/Core/CheckboxGroup.svelte b/packages/bbui/src/Form/Core/CheckboxGroup.svelte index 2b8a1e438a..d1a107fcc5 100644 --- a/packages/bbui/src/Form/Core/CheckboxGroup.svelte +++ b/packages/bbui/src/Form/Core/CheckboxGroup.svelte @@ -6,8 +6,8 @@ export let direction = "vertical" export let value = [] export let options = [] - export let error = null export let disabled = false + export let readonly = false export let getOptionLabel = option => option export let getOptionValue = option => option @@ -33,7 +33,7 @@