diff --git a/.eslintrc.json b/.eslintrc.json index 75584b8163..79f7e56712 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -19,6 +19,7 @@ "bundle.js" ], "extends": ["eslint:recommended"], + "plugins": ["import", "eslint-plugin-local-rules"], "overrides": [ { "files": ["**/*.svelte"], @@ -30,7 +31,6 @@ "sourceType": "module", "allowImportExportEverywhere": true } - }, { "files": ["**/*.ts"], @@ -42,13 +42,25 @@ "no-case-declarations": "off", "no-useless-escape": "off", "no-undef": "off", - "no-prototype-builtins": "off" + "no-prototype-builtins": "off", + "local-rules/no-budibase-imports": "error" } } ], "rules": { "no-self-assign": "off", - "no-unused-vars": ["error", { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", "destructuredArrayIgnorePattern": "^_" }] + "no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_" + } + ], + "import/no-relative-packages": "error", + "import/export": "error", + "import/no-duplicates": "error", + "import/newline-after-import": "error" }, "globals": { "GeolocationPositionError": true diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 77867c8617..6e04ca6f67 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -12,6 +12,13 @@ on: - master pull_request: workflow_dispatch: + workflow_call: + inputs: + run_as_oss: + type: boolean + required: false + description: Force running checks as if it was an OSS contributor + default: false env: BRANCH: ${{ github.event.pull_request.head.ref }} @@ -19,50 +26,41 @@ env: PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} NX_BASE_BRANCH: origin/${{ github.base_ref }} USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' }} + IS_OSS_CONTRIBUTOR: ${{ inputs.run_as_oss == true || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase') }} jobs: lint: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - run: yarn lint build: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile # Run build all the projects @@ -81,24 +79,18 @@ jobs: test-libraries: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -116,24 +108,18 @@ jobs: test-worker: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test worker run: | @@ -152,24 +138,18 @@ jobs: test-server: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - with: - fetch-depth: 0 - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test server run: | @@ -200,7 +180,7 @@ jobs: uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Test run: | @@ -213,24 +193,23 @@ jobs: integration-test: runs-on: ubuntu-latest steps: - - name: Checkout repo and submodules + - name: Checkout repo uses: actions/checkout@v3 - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' with: - submodules: true + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Checkout repo only - uses: actions/checkout@v3 - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase' - name: Use Node.js 18.x uses: actions/setup-node@v3 with: node-version: 18.x - cache: "yarn" + cache: yarn - run: yarn --frozen-lockfile - name: Build packages run: yarn build --scope @budibase/server --scope @budibase/worker + - name: Build backend-core for OSS contributor (required for pro) + if: ${{ env.IS_OSS_CONTRIBUTOR == 'true' }} + run: yarn build --scope @budibase/backend-core - name: Run tests run: | cd qa-core diff --git a/.github/workflows/check-oss-contributor.yml b/.github/workflows/check-oss-contributor.yml new file mode 100644 index 0000000000..398f07a130 --- /dev/null +++ b/.github/workflows/check-oss-contributor.yml @@ -0,0 +1,35 @@ +name: OSS contributor checks +on: + workflow_dispatch: + schedule: + - cron: "0 8,16 * * 1-5" # on weekdays at 8am and 4pm + +jobs: + run-checks: + name: Publish server and worker docker images + uses: ./.github/workflows/budibase_ci.yml + with: + run_as_oss: true + secrets: inherit + + notify-error: + needs: ["run-checks"] + if: ${{ failure() }} + name: Notify error + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set commit SHA + id: set_sha + run: echo "::set-output name=sha::$(git rev-parse --short ${{ github.sha }})" + + - name: Notify error + uses: tsickert/discord-webhook@v5.3.0 + with: + webhook-url: ${{ secrets.OSS_CHECKS_WEBHOOK_URL }} + embed-title: 🚨 OSS checks failed in master + embed-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + embed-description: | + Git sha: `${{ steps.set_sha.outputs.sha }}` diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml deleted file mode 100644 index 389b10f7d3..0000000000 --- a/.github/workflows/deploy-cloud.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Budibase Deploy Production - -on: - workflow_dispatch: - inputs: - version: - description: Budibase release version. For example - 1.0.0 - required: false - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - if [ -z "${{ github.event.inputs.version }}" ]; then - release_version=$(cat lerna.json | jq -r '.version') - else - release_version=${{ github.event.inputs.version }} - fi - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-prod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index f1fb12c087..c70f2fff20 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -7,6 +7,7 @@ on: jobs: release: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml deleted file mode 100644 index 2edb470405..0000000000 --- a/.github/workflows/release-master.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Budibase Release -concurrency: - group: release - cancel-in-progress: false - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - # Exclude all pre-releases - - "!*[0-9]+.[0-9]+.[0-9]+-*" - -env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - -jobs: - release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - cache: yarn - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default - git config --global user.name "Budibase Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Docker login - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - - name: Build worker docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/worker/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/worker - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build server docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/server/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/apps - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build proxy docker - uses: docker/build-push-action@v5 - with: - context: ./hosting/proxy - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./hosting/proxy/Dockerfile - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/proxy - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: peter-evans/repository-dispatch@v2 - with: - repository: budibase/budibase-deploys - event-type: budicloud-qa-deploy - token: ${{ secrets.GH_ACCESS_TOKEN }} - client-payload: |- - { - "VERSION": "${{ github.ref_name }}", - "REF_NAME": "${{ github.ref_name}}" - } diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml deleted file mode 100644 index d2689a0ea0..0000000000 --- a/.github/workflows/release-selfhost.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Budibase Release Selfhost - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Use Node.js 18.x - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - name: Tag and release Docker images (Self Host) - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - release_tag=${{ env.RELEASE_VERSION }} - - # Pull apps and worker images - docker pull budibase/apps:$release_tag - docker pull budibase/worker:$release_tag - docker pull budibase/proxy:$release_tag - - # Tag apps and worker images - docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG - docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG - docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG - - # Push images - docker push budibase/apps:$SELFHOST_TAG - docker push budibase/worker:$SELFHOST_TAG - docker push budibase/proxy:$SELFHOST_TAG - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - SELFHOST_TAG: latest - - - name: Bootstrap and build (CLI) - run: | - yarn - yarn build - - - name: Build OpenAPI spec - run: | - pushd packages/server - yarn - yarn specs - popd - - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Perform Github Release - uses: softprops/action-gh-release@v1 - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - generate_release_notes: true - files: | - packages/cli/build/cli-win.exe - packages/cli/build/cli-linux - packages/cli/build/cli-macos - packages/server/specs/openapi.yaml - packages/server/specs/openapi.json - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 - with: - webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml deleted file mode 100644 index 16b1da186a..0000000000 --- a/.github/workflows/release-singleimage.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Deploy Budibase Single Container Image to DockerHub - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 30000 - swap-size-mb: 1024 - remove-android: "true" - remove-dotnet: "true" - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - name: "Checkout" - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Run Yarn - run: yarn - - name: Update versions - run: ./scripts/updateVersions.sh - - name: Run Yarn Build - run: yarn build - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: | - TARGETBUILD=aas - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} diff --git a/.prettierignore b/.prettierignore index 64607d74ab..ce7617224b 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,13 +1,11 @@ node_modules dist -*.spec.js -packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte packages/server/builder packages/server/coverage -packages/worker/coverage -packages/backend-core/coverage packages/server/client packages/server/src/definitions/openapi.ts +packages/worker/coverage +packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk packages/pro/coverage \ No newline at end of file diff --git a/charts/budibase/templates/minio-service-deployment.yaml b/charts/budibase/templates/minio-service-deployment.yaml index 41af2624bf..f98ecc139d 100644 --- a/charts/budibase/templates/minio-service-deployment.yaml +++ b/charts/budibase/templates/minio-service-deployment.yaml @@ -46,11 +46,9 @@ spec: image: minio/minio imagePullPolicy: "" livenessProbe: - exec: - command: - - curl - - -f - - http://localhost:9000/minio/health/live + httpGet: + path: /minio/health/live + port: 9000 failureThreshold: 3 periodSeconds: 30 timeoutSeconds: 20 diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js new file mode 100644 index 0000000000..af02599c90 --- /dev/null +++ b/eslint-local-rules/index.js @@ -0,0 +1,21 @@ +module.exports = { + "no-budibase-imports": { + create: function (context) { + return { + ImportDeclaration(node) { + const importPath = node.source.value + + if ( + /^@budibase\/[^/]+\/.*$/.test(importPath) && + importPath !== "@budibase/backend-core/tests" + ) { + context.report({ + node, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`, + }) + } + }, + } + }, + }, +} diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index 792856cac7..f83df7038b 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -29,7 +29,6 @@ WORKDIR /opt/couchdb ADD couch/vm.args couch/local.ini ./etc/ WORKDIR / -ADD build-target-paths.sh . ADD runner.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/build-target-paths.sh b/hosting/couchdb/build-target-paths.sh deleted file mode 100644 index 34227011f4..0000000000 --- a/hosting/couchdb/build-target-paths.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -echo ${TARGETBUILD} > /buildtarget.txt -if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persistent data & SSH on port 2222 - DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - mkdir -p $DATA_DIR/{search,minio,couch} - mkdir -p $DATA_DIR/couch/{dbs,views} - chown -R couchdb:couchdb $DATA_DIR/couch/ - apt update - apt-get install -y openssh-server - echo "root:Docker!" | chpasswd - mkdir -p /tmp - chmod +x /tmp/ssh_setup.sh \ - && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) - cp /etc/sshd_config /etc/ssh/sshd_config - /etc/init.d/ssh restart - sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini -else - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -fi \ No newline at end of file diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index 4102d2a751..2e4d26122f 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -1,14 +1,52 @@ #!/bin/bash DATA_DIR=${DATA_DIR:-/data} + mkdir -p ${DATA_DIR} mkdir -p ${DATA_DIR}/couch/{dbs,views} mkdir -p ${DATA_DIR}/search chown -R couchdb:couchdb ${DATA_DIR}/couch -/build-target-paths.sh + +echo ${TARGETBUILD} > /buildtarget.txt +if [[ "${TARGETBUILD}" = "aas" ]]; then + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" + WEBSITES_ENABLE_APP_SERVICE_STORAGE=true + mkdir -p $DATA_DIR/{search,minio,couch} + mkdir -p $DATA_DIR/couch/{dbs,views} + chown -R couchdb:couchdb $DATA_DIR/couch/ + apt update + apt-get install -y openssh-server + echo "root:Docker!" | chpasswd + mkdir -p /tmp + chmod +x /tmp/ssh_setup.sh \ + && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) + cp /etc/sshd_config /etc/ssh/sshd_config + /etc/init.d/ssh restart + sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini +elif [[ "${TARGETBUILD}" = "single" ]]; then + sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then + # In Kubernetes the directory /opt/couchdb/data has a persistent volume + # mount for storing database data. + sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/couchdb/etc/local.ini + sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args +else + sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +fi + /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & -sleep 10 + +while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do + echo 'Waiting for CouchDB to start...'; + sleep 5; +done + curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator sleep infinity \ No newline at end of file diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index 7ead001a1c..dbc3613599 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -6,7 +6,7 @@ services: app-service: build: context: .. - dockerfile: packages/server/Dockerfile.v2 + dockerfile: packages/server/Dockerfile args: - BUDIBASE_VERSION=0.0.0+dev-docker container_name: build-bbapps @@ -36,7 +36,7 @@ services: worker-service: build: context: .. - dockerfile: packages/worker/Dockerfile.v2 + dockerfile: packages/worker/Dockerfile args: - BUDIBASE_VERSION=0.0.0+dev-docker container_name: build-bbworker diff --git a/hosting/scripts/build-target-paths.sh b/hosting/scripts/build-target-paths.sh deleted file mode 100644 index 34227011f4..0000000000 --- a/hosting/scripts/build-target-paths.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -echo ${TARGETBUILD} > /buildtarget.txt -if [[ "${TARGETBUILD}" = "aas" ]]; then - # Azure AppService uses /home for persistent data & SSH on port 2222 - DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - mkdir -p $DATA_DIR/{search,minio,couch} - mkdir -p $DATA_DIR/couch/{dbs,views} - chown -R couchdb:couchdb $DATA_DIR/couch/ - apt update - apt-get install -y openssh-server - echo "root:Docker!" | chpasswd - mkdir -p /tmp - chmod +x /tmp/ssh_setup.sh \ - && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) - cp /etc/sshd_config /etc/ssh/sshd_config - /etc/init.d/ssh restart - sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini -else - sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini - sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini -fi \ No newline at end of file diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index c7b90dbdc4..e9ff6c6596 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,44 +1,59 @@ FROM node:18-slim as build # install node-gyp dependencies -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python3 +RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq -# add pin script -WORKDIR / -ADD scripts/cleanup.sh ./ -RUN chmod +x /cleanup.sh -# build server +# copy and install dependencies WORKDIR /app -ADD packages/server . +COPY package.json . COPY yarn.lock . -RUN yarn install --production=true --network-timeout 1000000 -RUN /cleanup.sh +COPY lerna.json . +COPY .yarnrc . -# build worker -WORKDIR /worker -ADD packages/worker . -COPY yarn.lock . -RUN yarn install --production=true --network-timeout 1000000 -RUN /cleanup.sh +COPY packages/server/package.json packages/server/package.json +COPY packages/worker/package.json packages/worker/package.json +# string-templates does not get bundled during the esbuild process, so we want to use the local version +COPY packages/string-templates/package.json packages/string-templates/package.json -FROM budibase/couchdb + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json +RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json + + +# We will never want to sync pro, but the script is still required +RUN echo '' > scripts/syncProPackage.js +RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json +RUN ./scripts/removeWorkspaceDependencies.sh package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production + +# copy the actual code +COPY packages/server/dist packages/server/dist +COPY packages/server/pm2.config.js packages/server/pm2.config.js +COPY packages/server/client packages/server/client +COPY packages/server/builder packages/server/builder +COPY packages/worker/dist packages/worker/dist +COPY packages/worker/pm2.config.js packages/worker/pm2.config.js +COPY packages/string-templates packages/string-templates + + +FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single ENV TARGETBUILD $TARGETBUILD -COPY --from=build /app /app -COPY --from=build /worker /worker - # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -47,14 +62,12 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx -ADD hosting/single/nginx/nginx.conf /etc/nginx -ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default +COPY hosting/single/nginx/nginx.conf /etc/nginx +COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/nginx && \ touch /var/log/nginx/error.log && \ touch /var/run/nginx.pid && \ @@ -62,29 +75,39 @@ RUN mkdir -p /var/log/nginx && \ WORKDIR / RUN mkdir -p scripts/integrations/oracle -ADD packages/server/scripts/integrations/oracle scripts/integrations/oracle +COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh # setup minio WORKDIR /minio -ADD scripts/install-minio.sh ./install.sh +COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh # setup runner file WORKDIR / -ADD hosting/single/runner.sh . +COPY hosting/single/runner.sh . RUN chmod +x ./runner.sh -ADD hosting/single/healthcheck.sh . +COPY hosting/single/healthcheck.sh . RUN chmod +x ./healthcheck.sh # Script below sets the path for storing data based on $DATA_DIR # For Azure App Service install SSH & point data locations to /home -ADD hosting/single/ssh/sshd_config /etc/ -ADD hosting/single/ssh/ssh_setup.sh /tmp -RUN /build-target-paths.sh +COPY hosting/single/ssh/sshd_config /etc/ +COPY hosting/single/ssh/ssh_setup.sh /tmp + +# setup letsencrypt certificate +RUN apt-get install -y certbot python3-certbot-nginx +COPY hosting/letsencrypt /app/letsencrypt +RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh + +COPY --from=build /app/node_modules /node_modules +COPY --from=build /app/package.json /package.json +COPY --from=build /app/packages/server /app +COPY --from=build /app/packages/worker /worker +COPY --from=build /app/packages/string-templates /string-templates + +RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates -# cleanup cache -RUN yarn cache clean -f EXPOSE 80 EXPOSE 443 @@ -92,20 +115,10 @@ EXPOSE 443 EXPOSE 2222 VOLUME /data -# setup letsencrypt certificate -RUN apt-get install -y certbot python3-certbot-nginx -ADD hosting/letsencrypt /app/letsencrypt -RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh -# Remove cached files -RUN rm -rf \ - /root/.cache \ - /root/.npm \ - /root/.pip \ - /usr/local/share/doc \ - /usr/share/doc \ - /usr/share/man \ - /var/lib/apt/lists/* \ - /tmp/* +ARG BUDIBASE_VERSION +# Ensuring the version argument is sent +RUN test -n "$BUDIBASE_VERSION" +ENV BUDIBASE_VERSION=$BUDIBASE_VERSION HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 deleted file mode 100644 index ec03a1b5a2..0000000000 --- a/hosting/single/Dockerfile.v2 +++ /dev/null @@ -1,131 +0,0 @@ -FROM node:18-slim as build - -# install node-gyp dependencies -RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq - - -# copy and install dependencies -WORKDIR /app -COPY package.json . -COPY yarn.lock . -COPY lerna.json . -COPY .yarnrc . - -COPY packages/server/package.json packages/server/package.json -COPY packages/worker/package.json packages/worker/package.json -# string-templates does not get bundled during the esbuild process, so we want to use the local version -COPY packages/string-templates/package.json packages/string-templates/package.json - - -COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh -RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json -RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json - - -# We will never want to sync pro, but the script is still required -RUN echo '' > scripts/syncProPackage.js -RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json -RUN ./scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production - -# copy the actual code -COPY packages/server/dist packages/server/dist -COPY packages/server/pm2.config.js packages/server/pm2.config.js -COPY packages/server/client packages/server/client -COPY packages/server/builder packages/server/builder -COPY packages/worker/dist packages/worker/dist -COPY packages/worker/pm2.config.js packages/worker/pm2.config.js -COPY packages/string-templates packages/string-templates - - -FROM budibase/couchdb as runner -ARG TARGETARCH -ENV TARGETARCH $TARGETARCH -ENV NODE_MAJOR 18 -#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) -# e.g. docker build --build-arg TARGETBUILD=aas .... -ARG TARGETBUILD=single -ENV TARGETBUILD $TARGETBUILD - -# install base dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 - -# Install postgres client for pg_dump utils -RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ - && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ - && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ - && apt update -y \ - && apt install postgresql-client-15 -y \ - && apt remove software-properties-common apt-transport-https gpg -y - -# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx -WORKDIR /nodejs -COPY scripts/install-node.sh ./install.sh -RUN chmod +x install.sh && ./install.sh - -# setup nginx -COPY hosting/single/nginx/nginx.conf /etc/nginx -COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default -RUN mkdir -p /var/log/nginx && \ - touch /var/log/nginx/error.log && \ - touch /var/run/nginx.pid && \ - usermod -a -G tty www-data - -WORKDIR / -RUN mkdir -p scripts/integrations/oracle -COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle -RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh - -# setup minio -WORKDIR /minio -COPY scripts/install-minio.sh ./install.sh -RUN chmod +x install.sh && ./install.sh - -# setup runner file -WORKDIR / -COPY hosting/single/runner.sh . -RUN chmod +x ./runner.sh -COPY hosting/single/healthcheck.sh . -RUN chmod +x ./healthcheck.sh - -# Script below sets the path for storing data based on $DATA_DIR -# For Azure App Service install SSH & point data locations to /home -COPY hosting/single/ssh/sshd_config /etc/ -COPY hosting/single/ssh/ssh_setup.sh /tmp -RUN /build-target-paths.sh - - -# setup letsencrypt certificate -RUN apt-get install -y certbot python3-certbot-nginx -COPY hosting/letsencrypt /app/letsencrypt -RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh - -COPY --from=build /app/node_modules /node_modules -COPY --from=build /app/package.json /package.json -COPY --from=build /app/packages/server /app -COPY --from=build /app/packages/worker /worker -COPY --from=build /app/packages/string-templates /string-templates - -RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates - - -EXPOSE 80 -EXPOSE 443 -# Expose port 2222 for SSH on Azure App Service build -EXPOSE 2222 -VOLUME /data - -ARG BUDIBASE_VERSION -# Ensuring the version argument is sent -RUN test -n "$BUDIBASE_VERSION" -ENV BUDIBASE_VERSION=$BUDIBASE_VERSION - -HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" - -# must set this just before running -ENV NODE_ENV=production -WORKDIR / - -CMD ["./runner.sh"] diff --git a/hosting/single/healthcheck.sh b/hosting/single/healthcheck.sh index 592b3e94fa..12e340062c 100644 --- a/hosting/single/healthcheck.sh +++ b/hosting/single/healthcheck.sh @@ -25,7 +25,7 @@ if [[ $(curl -s -w "%{http_code}\n" http://localhost:4002/health -o /dev/null) - healthy=false fi -if [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/ -o /dev/null) -ne 200 ]]; then +if [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; then echo 'ERROR: CouchDB is not running'; healthy=false fi diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 87201c95c0..f4b2b5b127 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -22,11 +22,11 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME # Azure App Service customisations if [[ "${TARGETBUILD}" = "aas" ]]; then - DATA_DIR="${DATA_DIR:-/home}" + export DATA_DIR="${DATA_DIR:-/home}" WEBSITES_ENABLE_APP_SERVICE_STORAGE=true /etc/init.d/ssh start else - DATA_DIR=${DATA_DIR:-/data} + export DATA_DIR=${DATA_DIR:-/data} fi mkdir -p ${DATA_DIR} # Mount NFS or GCP Filestore if env vars exist for it diff --git a/lerna.json b/lerna.json index 5605642877..38872a31b1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.5", + "version": "2.13.18", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 8a27cde104..2978483448 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,17 @@ "name": "root", "private": true, "devDependencies": { + "@babel/core": "^7.22.5", + "@babel/eslint-parser": "^7.22.5", + "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", + "eslint-plugin-import": "^2.29.0", + "eslint-plugin-local-rules": "^2.0.0", + "eslint-plugin-svelte": "^2.32.2", "husky": "^8.0.3", "kill-port": "^1.6.1", "lerna": "7.1.1", @@ -17,12 +23,8 @@ "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "svelte": "3.49.0", - "typescript": "5.2.2", - "@babel/core": "^7.22.5", - "@babel/eslint-parser": "^7.22.5", - "@babel/preset-env": "^7.22.5", - "eslint-plugin-svelte": "^2.32.2", - "svelte-eslint-parser": "^0.32.0" + "svelte-eslint-parser": "^0.32.0", + "typescript": "5.2.2" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 0100a2d0e2..e31bc81eed 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -1,5 +1,6 @@ const _passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy + import { getGlobalDB } from "../context" import { Cookie } from "../constants" import { getSessionsForUser, invalidateSessions } from "../security/sessions" @@ -26,6 +27,7 @@ import { clearCookie, getCookie } from "../utils" import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso" const refresh = require("passport-oauth2-refresh") + export { auditLog, authError, diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index bd3efc20db..d442511fb8 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -19,7 +19,7 @@ async function populateFromDB(appId: string) { return doWithDB( appId, (db: Database) => { - return db.get(DocumentType.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7cd5d6227f..7a2be5a0f0 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -1,6 +1,6 @@ -const BaseCache = require("./base") +import BaseCache from "./base" -const GENERIC = new BaseCache.default() +const GENERIC = new BaseCache() export enum CacheKey { CHECKLIST = "checklist", @@ -19,6 +19,7 @@ export enum TTL { } function performExport(funcName: string) { + // @ts-ignore return (...args: any) => GENERIC[funcName](...args) } diff --git a/packages/backend-core/src/cache/index.ts b/packages/backend-core/src/cache/index.ts index 58928c271a..4fa986e4e2 100644 --- a/packages/backend-core/src/cache/index.ts +++ b/packages/backend-core/src/cache/index.ts @@ -2,4 +2,6 @@ export * as generic from "./generic" export * as user from "./user" export * as app from "./appMetadata" export * as writethrough from "./writethrough" +export * as invite from "./invite" +export * as passwordReset from "./passwordReset" export * from "./generic" diff --git a/packages/backend-core/src/cache/invite.ts b/packages/backend-core/src/cache/invite.ts new file mode 100644 index 0000000000..e43ebc4aa8 --- /dev/null +++ b/packages/backend-core/src/cache/invite.ts @@ -0,0 +1,86 @@ +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" +import env from "../environment" +import { getTenantId } from "../context" +import * as redis from "../redis/init" + +const TTL_SECONDS = Duration.fromDays(7).toSeconds() + +interface Invite { + email: string + info: any +} + +interface InviteWithCode extends Invite { + code: string +} + +/** + * Given an invite code and invite body, allow the update an existing/valid invite in redis + * @param code The invite code for an invite in redis + * @param value The body of the updated user invitation + */ +export async function updateCode(code: string, value: Invite) { + const client = await redis.getInviteClient() + await client.store(code, value, TTL_SECONDS) +} + +/** + * Generates an invitation code and writes it to redis - which can later be checked for user creation. + * @param email the email address which the code is being sent to (for use later). + * @param info Information to be carried along with the invitation. + * @return returns the code that was stored to redis. + */ +export async function createCode(email: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getInviteClient() + await client.store(code, { email, info }, TTL_SECONDS) + return code +} + +/** + * Checks that the provided invite code is valid - will return the email address of user that was invited. + * @param code the invite code that was provided as part of the link. + * @return If the code is valid then an email address will be returned. + */ +export async function getCode(code: string): Promise { + const client = await redis.getInviteClient() + const value = (await client.get(code)) as Invite | undefined + if (!value) { + throw "Invitation is not valid or has expired, please request a new one." + } + return value +} + +export async function deleteCode(code: string) { + const client = await redis.getInviteClient() + await client.delete(code) +} + +/** + Get all currently available user invitations for the current tenant. + **/ +export async function getInviteCodes(): Promise { + const client = await redis.getInviteClient() + const invites: { key: string; value: Invite }[] = await client.scan() + + const results: InviteWithCode[] = invites.map(invite => { + return { + ...invite.value, + code: invite.key, + } + }) + if (!env.MULTI_TENANCY) { + return results + } + const tenantId = getTenantId() + return results.filter(invite => tenantId === invite.info.tenantId) +} + +export async function getExistingInvites( + emails: string[] +): Promise { + return (await getInviteCodes()).filter(invite => + emails.includes(invite.email) + ) +} diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts new file mode 100644 index 0000000000..7f5a93f149 --- /dev/null +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -0,0 +1,38 @@ +import * as redis from "../redis/init" +import * as utils from "../utils" +import { Duration, DurationType } from "../utils" + +const TTL_SECONDS = Duration.fromHours(1).toSeconds() + +interface PasswordReset { + userId: string + info: any +} + +/** + * Given a user ID this will store a code (that is returned) for an hour in redis. + * The user can then return this code for resetting their password (through their reset link). + * @param userId the ID of the user which is to be reset. + * @param info Info about the user/the reset process. + * @return returns the code that was stored to redis. + */ +export async function createCode(userId: string, info: any): Promise { + const code = utils.newid() + const client = await redis.getPasswordResetClient() + await client.store(code, { userId, info }, TTL_SECONDS) + return code +} + +/** + * Given a reset code this will lookup to redis, check if the code is valid. + * @param code The code provided via the email link. + * @return returns the user ID if it is found + */ +export async function getCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + const value = (await client.get(code)) as PasswordReset | undefined + if (!value) { + throw "Provided information is not valid, cannot reset password - please try again." + } + return value +} diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0c83ed005d..0d189e3f7d 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -17,7 +17,6 @@ import { DocumentType, SEPARATOR } from "../constants" import { CacheKey, TTL, withCache } from "../cache" import * as context from "../context" import env from "../environment" -import environment from "../environment" // UTILS @@ -181,10 +180,10 @@ export async function getGoogleDatasourceConfig(): Promise< } export function getDefaultGoogleConfig(): GoogleInnerConfig | undefined { - if (environment.GOOGLE_CLIENT_ID && environment.GOOGLE_CLIENT_SECRET) { + if (env.GOOGLE_CLIENT_ID && env.GOOGLE_CLIENT_SECRET) { return { - clientID: environment.GOOGLE_CLIENT_ID!, - clientSecret: environment.GOOGLE_CLIENT_SECRET!, + clientID: env.GOOGLE_CLIENT_ID!, + clientSecret: env.GOOGLE_CLIENT_SECRET!, activated: true, } } diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index bb944556af..ac00483021 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -1,4 +1,5 @@ import { prefixed, DocumentType } from "@budibase/types" + export { SEPARATOR, UNICODE_MAX, diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index d29b6935a8..a59f5c6503 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -4,7 +4,7 @@ import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() - static run(context: ContextMap, func: any) { + static run(context: ContextMap, func: () => T) { return Context.storage.run(context, () => func()) } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 609c18abb5..d2259cfcab 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap { return context } -async function newContext(updates: ContextMap, task: any) { +async function newContext(updates: ContextMap, task: () => T) { // see if there already is a context setup let context: ContextMap = updateContext(updates) return Context.run(context, task) } -export async function doInAutomationContext(params: { +export async function doInAutomationContext(params: { appId: string automationId: string - task: any -}): Promise { + task: () => T +}): Promise { const tenantId = getTenantIDFromAppID(params.appId) return newContext( { @@ -144,10 +144,10 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext( +export async function doInAppContext( appId: string | null, - task: any -): Promise { + task: () => T +): Promise { if (!appId && !env.isTest()) { throw new Error("appId is required") } @@ -165,10 +165,10 @@ export async function doInAppContext( return newContext(updates, task) } -export async function doInIdentityContext( +export async function doInIdentityContext( identity: IdentityContext, - task: any -): Promise { + task: () => T +): Promise { if (!identity) { throw new Error("identity is required") } @@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database { */ export function getAppDB(opts?: any): Database { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve app DB - no app ID.") + } return getDB(appId, opts) } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 330b15e680..8588a7157a 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -10,6 +10,7 @@ import { DatabaseDeleteIndexOpts, Document, isDocument, + RowResponse, } from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" @@ -48,10 +49,7 @@ export class DatabaseImpl implements Database { private readonly couchInfo = getCouchInfo() - constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { - if (dbName == null) { - throw new Error("Database name cannot be undefined.") - } + constructor(dbName: string, opts?: DatabaseOpts, connection?: string) { this.name = dbName this.pouchOpts = opts || {} if (connection) { @@ -112,7 +110,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -120,6 +118,35 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const rowUnavailable = (row: RowResponse) => { + // row is deleted - key lookup can return this + if (row.doc == null || ("deleted" in row.value && row.value.deleted)) { + return true + } + return row.error === "not_found" + } + + const rows = response.rows.filter(row => !rowUnavailable(row)) + const someMissing = rows.length !== response.rows.length + // some were filtered out - means some missing + if (!opts?.allowMissing && someMissing) { + const missing = response.rows.filter(row => rowUnavailable(row)) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 9aae64b892..3e69d49f0e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,10 +1,7 @@ -import env from "../environment" import { directCouchQuery, DatabaseImpl } from "./couch" -import { CouchFindOptions, Database } from "@budibase/types" +import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" -const dbList = new Set() - -export function getDB(dbName?: string, opts?: any): Database { +export function getDB(dbName: string, opts?: DatabaseOpts): Database { return new DatabaseImpl(dbName, opts) } @@ -14,7 +11,7 @@ export function getDB(dbName?: string, opts?: any): Database { export async function doWithDB( dbName: string, cb: (db: Database) => Promise, - opts = {} + opts?: DatabaseOpts ) { const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all @@ -22,13 +19,6 @@ export async function doWithDB( return await cb(db) } -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - return [...dbList] -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/backend-core/src/db/tests/index.spec.js b/packages/backend-core/src/db/tests/index.spec.js index 0d257f7ed7..e03c9a5b0e 100644 --- a/packages/backend-core/src/db/tests/index.spec.js +++ b/packages/backend-core/src/db/tests/index.spec.js @@ -5,7 +5,6 @@ const { getDB } = require("../db") describe("db", () => { describe("getDB", () => { it("returns a db", async () => { - const dbName = structures.db.id() const db = getDB(dbName) expect(db).toBeDefined() diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index d7a4b8224a..906a95e1db 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -6,6 +6,7 @@ import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata" import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions" import { App, Database } from "@budibase/types" import { getStartEndKeyURL } from "../docIds" + export * from "../docIds" /** diff --git a/packages/backend-core/src/docIds/conversions.ts b/packages/backend-core/src/docIds/conversions.ts index b168b74e16..ec43d01389 100644 --- a/packages/backend-core/src/docIds/conversions.ts +++ b/packages/backend-core/src/docIds/conversions.ts @@ -1,5 +1,6 @@ import { APP_DEV_PREFIX, APP_PREFIX } from "../constants" import { App } from "@budibase/types" + const NO_APP_ERROR = "No app provided" export function isDevAppID(appId?: string) { diff --git a/packages/backend-core/src/events/processors/posthog/index.ts b/packages/backend-core/src/events/processors/posthog/index.ts index dceb10d2cd..5a2b1afc9f 100644 --- a/packages/backend-core/src/events/processors/posthog/index.ts +++ b/packages/backend-core/src/events/processors/posthog/index.ts @@ -1,2 +1,3 @@ import PosthogProcessor from "./PosthogProcessor" + export default PosthogProcessor diff --git a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts index 0722fc3293..d9a5504073 100644 --- a/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts +++ b/packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts @@ -1,7 +1,9 @@ import { testEnv } from "../../../../../tests/extra" import PosthogProcessor from "../PosthogProcessor" import { Event, IdentityType, Hosting } from "@budibase/types" + const tk = require("timekeeper") + import * as cache from "../../../../cache/generic" import { CacheKey } from "../../../../cache/generic" import * as context from "../../../../context" diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 8f5c903e05..ad517082de 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -1,5 +1,6 @@ import env from "../environment" import * as context from "../context" + export * from "./installation" /** diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index c7cf9f56cc..d04f48e5fc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -32,11 +32,13 @@ export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" export * from "./utils/Duration" export { SearchParams } from "./db" +export * as docIds from "./docIds" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies import * as context from "./context" import * as _tenancy from "./tenancy" + export const tenancy = { ..._tenancy, ...context, @@ -50,6 +52,7 @@ export * from "./constants" // expose package init function import * as db from "./db" + export const init = (opts: any = {}) => { db.init(opts.db) } diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts index 17eda2004d..ca35b926fb 100644 --- a/packages/backend-core/src/installation.ts +++ b/packages/backend-core/src/installation.ts @@ -1,7 +1,6 @@ import { newid } from "./utils" import * as events from "./events" -import { StaticDatabases } from "./db" -import { doWithDB } from "./db" +import { StaticDatabases, doWithDB } from "./db" import { Installation, IdentityType, Database } from "@budibase/types" import * as context from "./context" import semver from "semver" diff --git a/packages/backend-core/src/logging/correlation/correlation.ts b/packages/backend-core/src/logging/correlation/correlation.ts index b5b47df9c6..0498bd43d5 100644 --- a/packages/backend-core/src/logging/correlation/correlation.ts +++ b/packages/backend-core/src/logging/correlation/correlation.ts @@ -1,4 +1,5 @@ import { Header } from "../../constants" + const correlator = require("correlation-id") export const setHeader = (headers: any) => { diff --git a/packages/backend-core/src/logging/correlation/middleware.ts b/packages/backend-core/src/logging/correlation/middleware.ts index f77714a5ae..45baee1fb1 100644 --- a/packages/backend-core/src/logging/correlation/middleware.ts +++ b/packages/backend-core/src/logging/correlation/middleware.ts @@ -1,5 +1,6 @@ import { Header } from "../../constants" import { v4 as uuid } from "uuid" + const correlator = require("correlation-id") const correlation = (ctx: any, next: any) => { diff --git a/packages/backend-core/src/logging/pino/middleware.ts b/packages/backend-core/src/logging/pino/middleware.ts index 569420c5f2..df18a35eb1 100644 --- a/packages/backend-core/src/logging/pino/middleware.ts +++ b/packages/backend-core/src/logging/pino/middleware.ts @@ -1,9 +1,12 @@ import env from "../../environment" import { logger } from "./logger" import { IncomingMessage } from "http" + const pino = require("koa-pino-logger") + import { Options } from "pino-http" import { Ctx } from "@budibase/types" + const correlator = require("correlation-id") export function pinoSettings(): Options { diff --git a/packages/backend-core/src/middleware/index.ts b/packages/backend-core/src/middleware/index.ts index 980bf06b00..e1eb7f1d26 100644 --- a/packages/backend-core/src/middleware/index.ts +++ b/packages/backend-core/src/middleware/index.ts @@ -2,6 +2,7 @@ export * as local from "./passport/local" export * as google from "./passport/sso/google" export * as oidc from "./passport/sso/oidc" import * as datasourceGoogle from "./passport/datasource/google" + export const datasource = { google: datasourceGoogle, } diff --git a/packages/backend-core/src/middleware/passport/sso/google.ts b/packages/backend-core/src/middleware/passport/sso/google.ts index ad7593e63d..2a08ad7665 100644 --- a/packages/backend-core/src/middleware/passport/sso/google.ts +++ b/packages/backend-core/src/middleware/passport/sso/google.ts @@ -8,6 +8,7 @@ import { SaveSSOUserFunction, GoogleInnerConfig, } from "@budibase/types" + const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) { diff --git a/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts index d0689a1f0a..9bf855b3c5 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts @@ -6,6 +6,7 @@ const mockStrategy = require("passport-google-oauth").OAuth2Strategy jest.mock("../sso") import * as _sso from "../sso" + const sso = jest.mocked(_sso) const mockSaveUserFn = jest.fn() diff --git a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts index c3ddf220e6..d3486a5b14 100644 --- a/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts +++ b/packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts @@ -11,6 +11,7 @@ const mockSaveUser = jest.fn() jest.mock("../../../../users") import * as _users from "../../../../users" + const users = jest.mocked(_users) const getErrorMessage = () => { diff --git a/packages/backend-core/src/middleware/tests/builder.spec.ts b/packages/backend-core/src/middleware/tests/builder.spec.ts index d350eff4f6..0514dc13f0 100644 --- a/packages/backend-core/src/middleware/tests/builder.spec.ts +++ b/packages/backend-core/src/middleware/tests/builder.spec.ts @@ -5,6 +5,7 @@ import { structures } from "../../../tests" import { ContextUser, ServiceType } from "@budibase/types" import { doInAppContext } from "../../context" import env from "../../environment" + env._set("SERVICE_TYPE", ServiceType.APPS) const appId = "app_aaa" diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 76d2dd6689..1971c09e9d 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,4 +1,5 @@ const sanitize = require("sanitize-s3-objectkey") + import AWS from "aws-sdk" import stream, { Readable } from "stream" import fetch from "node-fetch" @@ -259,12 +260,12 @@ export async function listAllObjects(bucketName: string, path: string) { } /** - * Generate a presigned url with a default TTL of 1 hour + * Generate a presigned url with a default TTL of 36 hours */ export function getPresignedUrl( bucketName: string, key: string, - durationSeconds: number = 3600 + durationSeconds: number = 129600 ) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index 55ffe3dd12..f3bcee3209 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -7,15 +7,19 @@ let userClient: Client, cacheClient: Client, writethroughClient: Client, lockClient: Client, - socketClient: Client + socketClient: Client, + inviteClient: Client, + passwordResetClient: Client -async function init() { +export async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() sessionClient = await new Client(utils.Databases.SESSIONS).init() appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() lockClient = await new Client(utils.Databases.LOCKS).init() writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init() + inviteClient = await new Client(utils.Databases.INVITATIONS).init() + passwordResetClient = await new Client(utils.Databases.PW_RESETS).init() socketClient = await new Client( utils.Databases.SOCKET_IO, utils.SelectableDatabase.SOCKET_IO @@ -29,6 +33,8 @@ export async function shutdown() { if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() if (lockClient) await lockClient.finish() + if (inviteClient) await inviteClient.finish() + if (passwordResetClient) await passwordResetClient.finish() if (socketClient) await socketClient.finish() } @@ -84,3 +90,17 @@ export async function getSocketClient() { } return socketClient } + +export async function getInviteClient() { + if (!inviteClient) { + await init() + } + return inviteClient +} + +export async function getPasswordResetClient() { + if (!passwordResetClient) { + await init() + } + return passwordResetClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 6f1b573718..701e262091 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -28,7 +28,6 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false let CLIENTS: { [key: number]: any } = {} -0 let CONNECTED = false // mock redis always connected diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 7fe61a409e..266f1fe989 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -3,6 +3,7 @@ import { getLockClient } from "./init" import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" import env from "../environment" +import { logWarn } from "../logging" async function getClient( type: LockType, @@ -116,7 +117,7 @@ export async function doWithLock( const result = await task() return { executed: true, result } } catch (e: any) { - console.warn("lock error") + logWarn(`lock type: ${opts.type} error`, e) // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { @@ -124,11 +125,9 @@ export async function doWithLock( // due to retry count (0) exceeded return { executed: false } } else { - console.error(e) throw e } } else { - console.error(e) throw e } } finally { diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 5187fe13f8..4d8b1bb9a4 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -75,10 +75,12 @@ export function getRedisConnectionDetails() { } const [host, port] = url.split(":") + const portNumber = parseInt(port) return { host, password, - port: parseInt(port), + // assume default port for redis if invalid found + port: isNaN(portNumber) ? 6379 : portNumber, } } diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index fe4095d210..98704f16c6 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -160,4 +160,5 @@ export function isPermissionLevelHigherThanRead(level: PermissionLevel) { // utility as a lot of things need simply the builder permission export const BUILDER = PermissionType.BUILDER +export const CREATOR = PermissionType.CREATOR export const GLOBAL_BUILDER = PermissionType.GLOBAL_BUILDER diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 0d33031de5..4f048c0a11 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,7 +1,12 @@ import { BuiltinPermissionID, PermissionLevel } from "./permissions" -import { prefixRoleID, getRoleParams, DocumentType, SEPARATOR } from "../db" +import { + prefixRoleID, + getRoleParams, + DocumentType, + SEPARATOR, + doWithDB, +} from "../db" import { getAppDB } from "../context" -import { doWithDB } from "../db" import { Screen, Role as RoleDoc } from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" diff --git a/packages/backend-core/src/security/sessions.ts b/packages/backend-core/src/security/sessions.ts index 5a535c0c46..a86a829b17 100644 --- a/packages/backend-core/src/security/sessions.ts +++ b/packages/backend-core/src/security/sessions.ts @@ -1,6 +1,7 @@ const redis = require("../redis/init") const { v4: uuidv4 } = require("uuid") const { logWarn } = require("../logging") + import env from "../environment" import { Session, diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 59f698d99c..326bed3cc5 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -1,9 +1,8 @@ import env from "../environment" import * as eventHelpers from "./events" -import * as accounts from "../accounts" import * as accountSdk from "../accounts" import * as cache from "../cache" -import { getGlobalDB, getIdentity, getTenantId } from "../context" +import { doInTenant, getGlobalDB, getIdentity, getTenantId } from "../context" import * as dbUtils from "../db" import { EmailUnavailableError, HTTPError } from "../errors" import * as platform from "../platform" @@ -11,12 +10,10 @@ import * as sessions from "../security/sessions" import * as usersCore from "./users" import { Account, - AllDocsResponse, BulkUserCreated, BulkUserDeleted, isSSOAccount, isSSOUser, - RowResponse, SaveUserOpts, User, UserStatus, @@ -149,12 +146,12 @@ export class UserDB { static async allUsers() { const db = getGlobalDB() - const response = await db.allDocs( + const response = await db.allDocs( dbUtils.getGlobalUserParams(null, { include_docs: true, }) ) - return response.rows.map((row: any) => row.doc) + return response.rows.map(row => row.doc!) } static async countUsersByApp(appId: string) { @@ -212,13 +209,6 @@ export class UserDB { throw new Error("_id or email is required") } - if ( - user.builder?.apps?.length && - !(await UserDB.features.isAppBuildersEnabled()) - ) { - throw new Error("Unable to update app builders, please check license") - } - let dbUser: User | undefined if (_id) { // try to get existing user from db @@ -303,7 +293,7 @@ export class UserDB { static async bulkCreate( newUsersRequested: User[], - groups: string[] + groups?: string[] ): Promise { const tenantId = getTenantId() @@ -328,7 +318,7 @@ export class UserDB { }) continue } - newUser.userGroups = groups + newUser.userGroups = groups || [] newUsers.push(newUser) if (isCreator(newUser)) { newCreators.push(newUser) @@ -467,7 +457,7 @@ export class UserDB { if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { // root account holder can't be deleted from inside budibase const email = dbUser.email - const account = await accounts.getAccount(email) + const account = await accountSdk.getAccount(email) if (account) { if (dbUser.userId === getIdentity()!._id) { throw new HTTPError('Please visit "Account" to delete this user', 400) @@ -488,6 +478,37 @@ export class UserDB { await sessions.invalidateSessions(userId, { reason: "deletion" }) } + static async createAdminUser( + email: string, + password: string, + tenantId: string, + opts?: { ssoId?: string; hashPassword?: boolean; requirePassword?: boolean } + ) { + const user: User = { + email: email, + password: password, + createdAt: Date.now(), + roles: {}, + builder: { + global: true, + }, + admin: { + global: true, + }, + tenantId, + } + if (opts?.ssoId) { + user.ssoId = opts.ssoId + } + // always bust checklist beforehand, if an error occurs but can proceed, don't get + // stuck in a cycle + await cache.bustCache(cache.CacheKey.CHECKLIST) + return await UserDB.save(user, { + hashPassword: opts?.hashPassword, + requirePassword: opts?.requirePassword, + }) + } + static async getGroups(groupIds: string[]) { return await this.groups.getBulk(groupIds) } diff --git a/packages/backend-core/src/users/lookup.ts b/packages/backend-core/src/users/lookup.ts index 17d0e91d88..355be74dab 100644 --- a/packages/backend-core/src/users/lookup.ts +++ b/packages/backend-core/src/users/lookup.ts @@ -6,6 +6,7 @@ import { } from "@budibase/types" import * as dbUtils from "../db" import { ViewName } from "../constants" +import { getExistingInvites } from "../cache/invite" /** * Apply a system-wide search on emails: @@ -26,6 +27,9 @@ export async function searchExistingEmails(emails: string[]) { const existingAccounts = await getExistingAccounts(emails) matchedEmails.push(...existingAccounts.map(account => account.email)) + const invitedEmails = await getExistingInvites(emails) + matchedEmails.push(...invitedEmails.map(invite => invite.email)) + return [...new Set(matchedEmails.map(email => email.toLowerCase()))] } diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 9f4a41f6df..cc2b4fc27f 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -25,6 +25,7 @@ import { import { getGlobalDB } from "../context" import * as context from "../context" import { isCreator } from "./utils" +import { UserDB } from "./db" type GetOpts = { cleanup?: boolean } @@ -43,7 +44,7 @@ function removeUserPassword(users: User | User[]) { return users } -export const isSupportedUserSearch = (query: SearchQuery) => { +export function isSupportedUserSearch(query: SearchQuery) { const allowed = [ { op: SearchQueryOperators.STRING, key: "email" }, { op: SearchQueryOperators.EQUAL, key: "_id" }, @@ -68,10 +69,10 @@ export const isSupportedUserSearch = (query: SearchQuery) => { return true } -export const bulkGetGlobalUsersById = async ( +export async function bulkGetGlobalUsersById( userIds: string[], opts?: GetOpts -) => { +) { const db = getGlobalDB() let users = ( await db.allDocs({ @@ -85,7 +86,7 @@ export const bulkGetGlobalUsersById = async ( return users } -export const getAllUserIds = async () => { +export async function getAllUserIds() { const db = getGlobalDB() const startKey = `${DocumentType.USER}${SEPARATOR}` const response = await db.allDocs({ @@ -95,7 +96,7 @@ export const getAllUserIds = async () => { return response.rows.map(row => row.id) } -export const bulkUpdateGlobalUsers = async (users: User[]) => { +export async function bulkUpdateGlobalUsers(users: User[]) { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse } @@ -113,10 +114,10 @@ export async function getById(id: string, opts?: GetOpts): Promise { * Given an email address this will use a view to search through * all the users to find one with this email address. */ -export const getGlobalUserByEmail = async ( +export async function getGlobalUserByEmail( email: String, opts?: GetOpts -): Promise => { +): Promise { if (email == null) { throw "Must supply an email address to view" } @@ -139,11 +140,23 @@ export const getGlobalUserByEmail = async ( return user } -export const searchGlobalUsersByApp = async ( +export async function doesUserExist(email: string) { + try { + const user = await getGlobalUserByEmail(email) + if (Array.isArray(user) || user != null) { + return true + } + } catch (err) { + return false + } + return false +} + +export async function searchGlobalUsersByApp( appId: any, opts: DatabaseQueryOpts, getOpts?: GetOpts -) => { +) { if (typeof appId !== "string") { throw new Error("Must provide a string based app ID") } @@ -167,10 +180,10 @@ export const searchGlobalUsersByApp = async ( Return any user who potentially has access to the application Admins, developers and app users with the explicitly role. */ -export const searchGlobalUsersByAppAccess = async ( +export async function searchGlobalUsersByAppAccess( appId: any, opts?: { limit?: number } -) => { +) { const roleSelector = `roles.${appId}` let orQuery: any[] = [ @@ -205,7 +218,7 @@ export const searchGlobalUsersByAppAccess = async ( return resp.rows } -export const getGlobalUserByAppPage = (appId: string, user: User) => { +export function getGlobalUserByAppPage(appId: string, user: User) { if (!user) { return } @@ -215,11 +228,11 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => { /** * Performs a starts with search on the global email view. */ -export const searchGlobalUsersByEmail = async ( +export async function searchGlobalUsersByEmail( email: string | unknown, opts: any, getOpts?: GetOpts -) => { +) { if (typeof email !== "string") { throw new Error("Must provide a string to search by") } @@ -242,12 +255,12 @@ export const searchGlobalUsersByEmail = async ( } const PAGE_LIMIT = 8 -export const paginatedUsers = async ({ +export async function paginatedUsers({ bookmark, query, appId, limit, -}: SearchUsersRequest = {}) => { +}: SearchUsersRequest = {}) { const db = getGlobalDB() const pageSize = limit ?? PAGE_LIMIT const pageLimit = pageSize + 1 @@ -324,3 +337,20 @@ export function cleanseUserObject(user: User | ContextUser, base?: User) { } return user } + +export async function addAppBuilder(user: User, appId: string) { + const prodAppId = getProdAppID(appId) + user.builder ??= {} + user.builder.creator = true + user.builder.apps ??= [] + user.builder.apps.push(prodAppId) + await UserDB.save(user, { hashPassword: false }) +} + +export async function removeAppBuilder(user: User, appId: string) { + const prodAppId = getProdAppID(appId) + if (user.builder && user.builder.apps?.includes(prodAppId)) { + user.builder.apps = user.builder.apps.filter(id => id !== prodAppId) + } + await UserDB.save(user, { hashPassword: false }) +} diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts index f376c2f7c7..3c7ef23b11 100644 --- a/packages/backend-core/src/utils/Duration.ts +++ b/packages/backend-core/src/utils/Duration.ts @@ -28,6 +28,9 @@ export class Duration { toMs: () => { return Duration.convert(from, DurationType.MILLISECONDS, duration) }, + toSeconds: () => { + return Duration.convert(from, DurationType.SECONDS, duration) + }, } } diff --git a/packages/backend-core/src/utils/hashing.ts b/packages/backend-core/src/utils/hashing.ts index aba11f38e6..54d7de4aba 100644 --- a/packages/backend-core/src/utils/hashing.ts +++ b/packages/backend-core/src/utils/hashing.ts @@ -1,4 +1,5 @@ import env from "../environment" + export * from "../docIds/newid" const bcrypt = env.JS_BCRYPT ? require("bcryptjs") : require("bcrypt") diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 1c1ca8473b..b10d9ebdc0 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -11,6 +11,7 @@ import { TenantResolutionStrategy, } from "@budibase/types" import type { SetOption } from "cookies" + const jwt = require("jsonwebtoken") const APP_PREFIX = DocumentType.APP + SEPARATOR diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js index ae7109344a..f08c435b95 100644 --- a/packages/backend-core/tests/core/users/users.spec.js +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -1,5 +1,5 @@ -const _ = require('lodash/fp') -const {structures} = require("../../../tests") +const _ = require("lodash/fp") +const { structures } = require("../../../tests") jest.mock("../../../src/context") jest.mock("../../../src/db") @@ -7,10 +7,9 @@ jest.mock("../../../src/db") const context = require("../../../src/context") const db = require("../../../src/db") -const {getCreatorCount} = require('../../../src/users/users') +const { getCreatorCount } = require("../../../src/users/users") describe("Users", () => { - let getGlobalDBMock let getGlobalUserParamsMock let paginationMock @@ -26,26 +25,26 @@ describe("Users", () => { it("Retrieves the number of creators", async () => { const getUsers = (offset, limit, creators = false) => { const range = _.range(offset, limit) - const opts = creators ? {builder: {global: true}} : undefined + const opts = creators ? { builder: { global: true } } : undefined return range.map(() => structures.users.user(opts)) } const page1Data = getUsers(0, 8) const page2Data = getUsers(8, 12, true) getGlobalDBMock.mockImplementation(() => ({ - name : "fake-db", + name: "fake-db", allDocs: () => ({ - rows: [...page1Data, ...page2Data] - }) + rows: [...page1Data, ...page2Data], + }), })) paginationMock.mockImplementationOnce(() => ({ data: page1Data, hasNextPage: true, - nextPage: "1" + nextPage: "1", })) paginationMock.mockImplementation(() => ({ data: page2Data, hasNextPage: false, - nextPage: undefined + nextPage: undefined, })) const creatorsCount = await getCreatorCount() expect(creatorsCount).toBe(4) diff --git a/packages/backend-core/tests/core/utilities/mocks/alerts.ts b/packages/backend-core/tests/core/utilities/mocks/alerts.ts index 90c9759c92..0b26e98363 100644 --- a/packages/backend-core/tests/core/utilities/mocks/alerts.ts +++ b/packages/backend-core/tests/core/utilities/mocks/alerts.ts @@ -1,3 +1,4 @@ jest.mock("../../../../src/logging/alerts") import * as _alerts from "../../../../src/logging/alerts" + export const alerts = jest.mocked(_alerts) diff --git a/packages/backend-core/tests/core/utilities/mocks/index.ts b/packages/backend-core/tests/core/utilities/mocks/index.ts index 9a72b38ef5..8705e563cb 100644 --- a/packages/backend-core/tests/core/utilities/mocks/index.ts +++ b/packages/backend-core/tests/core/utilities/mocks/index.ts @@ -1,5 +1,6 @@ jest.mock("../../../../src/accounts") import * as _accounts from "../../../../src/accounts" + export const accounts = jest.mocked(_accounts) export * as date from "./date" diff --git a/packages/backend-core/tests/core/utilities/structures/generator.ts b/packages/backend-core/tests/core/utilities/structures/generator.ts index ed4dac8255..64eb5ecc97 100644 --- a/packages/backend-core/tests/core/utilities/structures/generator.ts +++ b/packages/backend-core/tests/core/utilities/structures/generator.ts @@ -1,2 +1,3 @@ import Chance from "./Chance" + export const generator = new Chance() diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 66d23696e0..68ee29686c 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -12,7 +12,7 @@ import { generator } from "./generator" import { tenant } from "." export const newEmail = () => { - return `${uuid()}@test.com` + return `${uuid()}@example.com` } export const user = (userProps?: Partial>): User => { diff --git a/packages/backend-core/tests/jestSetup.ts b/packages/backend-core/tests/jestSetup.ts index 42a24ce733..e5d144290b 100644 --- a/packages/backend-core/tests/jestSetup.ts +++ b/packages/backend-core/tests/jestSetup.ts @@ -9,6 +9,7 @@ mocks.fetch.enable() // mock all dates to 2020-01-01T00:00:00.000Z // use tk.reset() to use real dates in individual tests import tk from "timekeeper" + tk.freeze(mocks.date.MOCK_DATE) if (!process.env.DEBUG) { diff --git a/packages/bbui/src/ActionGroup/ActionGroup.svelte b/packages/bbui/src/ActionGroup/ActionGroup.svelte index 43d8cd8de5..978e920c42 100644 --- a/packages/bbui/src/ActionGroup/ActionGroup.svelte +++ b/packages/bbui/src/ActionGroup/ActionGroup.svelte @@ -1,5 +1,6 @@ - + diff --git a/packages/bbui/src/Form/Combobox.svelte b/packages/bbui/src/Form/Combobox.svelte index 343af559cb..44854d949e 100644 --- a/packages/bbui/src/Form/Combobox.svelte +++ b/packages/bbui/src/Form/Combobox.svelte @@ -11,6 +11,7 @@ export let error = null export let placeholder = "Choose an option or type" export let options = [] + export let helpText = null export let getOptionLabel = option => extractProperty(option, "label") export let getOptionValue = option => extractProperty(option, "value") @@ -27,7 +28,7 @@ } - + option @@ -34,7 +33,6 @@