diff --git a/.dockerignore b/.dockerignore index 92bd33894e..86e1f739f7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,14 @@ -packages/server/node_modules -packages/builder -packages/frontend-core -packages/backend-core -packages/worker/node_modules -packages/cli -packages/client -packages/bbui -packages/string-templates +* +!/packages/ +!/scripts/ +/packages/*/node_modules +packages/server/scripts/ +!packages/server/scripts/integrations/oracle +!nx.json +!/hosting/single/ +!/hosting/letsencrypt / + +!package.json +!yarn.lock +!lerna.json +!.yarnrc \ No newline at end of file diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index bb64f5ce20..46cb781730 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -20,7 +20,7 @@ jobs: - uses: passeidireto/trigger-external-workflow-action@main env: PAYLOAD_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.BRANCH || github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} with: repository: budibase/budibase-deploys event: featurebranch-qa-close diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 3af3a751ad..4acd5088d2 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -14,7 +14,6 @@ env: # Posthog token used by ui at build time POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: @@ -110,7 +109,6 @@ jobs: git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" git push - trigger-deploy-to-qa-env: needs: [release-helm-chart] runs-on: ubuntu-latest diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml new file mode 100644 index 0000000000..79b9afdd44 --- /dev/null +++ b/.github/workflows/release-singleimage-test.yml @@ -0,0 +1,69 @@ +name: Test + +on: + workflow_dispatch: + +env: + CI: true + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} +jobs: + build: + name: "build" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Run Yarn + run: yarn + - name: Run Yarn Build + run: yarn build --scope @budibase/server --scope @budibase/worker + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Budibase service docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + pull: true + platforms: linux/amd64,linux/arm64 + tags: budibase/budibase-test:test + file: ./hosting/single/Dockerfile.v2 + cache-from: type=registry,ref=budibase/budibase-test:test + cache-to: type=inline + - name: Tag and release Budibase Azure App Service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64 + build-args: TARGETBUILD=aas + tags: budibase/budibase-test:aas + file: ./hosting/single/Dockerfile.v2 diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 61ab9a4eb2..f7f87f6e4c 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -20,8 +20,8 @@ jobs: with: root-reserve-mb: 30000 swap-size-mb: 1024 - remove-android: 'true' - remove-dotnet: 'true' + remove-android: "true" + remove-dotnet: "true" - name: Fail if not a tag run: | if [[ $GITHUB_REF != refs/tags/* ]]; then @@ -48,7 +48,7 @@ jobs: - name: Update versions run: ./scripts/updateVersions.sh - name: Run Yarn Build - run: yarn build:docker:pre + run: yarn build - name: Login to Docker Hub uses: docker/login-action@v2 with: diff --git a/.yarnrc b/.yarnrc index 21fa517e23..a3a3d23ec6 100644 --- a/.yarnrc +++ b/.yarnrc @@ -1 +1 @@ -network-timeout 100000 +network-timeout 1000000 diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 3243509094..73c6d990d2 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -134,8 +134,6 @@ spec: {{ end }} - name: SELF_HOSTED value: {{ .Values.globals.selfHosted | quote }} - - name: SENTRY_DSN - value: {{ .Values.globals.sentryDSN | quote }} - name: POSTHOG_TOKEN value: {{ .Values.globals.posthogToken | quote }} - name: WORKER_URL diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 7621aa23ef..5e0addb9dd 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -130,8 +130,6 @@ spec: {{ end }} - name: SELF_HOSTED value: {{ .Values.globals.selfHosted | quote }} - - name: SENTRY_DSN - value: {{ .Values.globals.sentryDSN }} - name: ENABLE_ANALYTICS value: {{ .Values.globals.enableAnalytics | quote }} - name: POSTHOG_TOKEN diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index e5f1eabb53..857067d0f1 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -78,7 +78,6 @@ globals: budibaseEnv: PRODUCTION tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" enableAnalytics: "1" - sentryDSN: "" posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU" selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 3a32075a33..77afd9453b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -138,6 +138,8 @@ To develop the Budibase platform you'll need [Docker](https://www.docker.com/) a `yarn setup` will check that all necessary components are installed and setup the repo for usage. +If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above command. + ##### Manual method The following commands can be executed to manually get Budibase up and running (assuming Docker/Docker Compose has been installed). @@ -146,6 +148,8 @@ The following commands can be executed to manually get Budibase up and running ( `yarn build` will build all budibase packages. +If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above commands. + #### 4. Running To run the budibase server and builder in dev mode (i.e. with live reloading): diff --git a/hosting/couchdb/couch/local.ini b/hosting/couchdb/couch/local.ini index 266c0d4b60..34e74dd72a 100644 --- a/hosting/couchdb/couch/local.ini +++ b/hosting/couchdb/couch/local.ini @@ -3,3 +3,6 @@ [couchdb] database_dir = DATA_DIR/couch/dbs view_index_dir = DATA_DIR/couch/views + +[chttpd_auth] +timeout = 7200 ; 2 hours in seconds diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml index 391d263098..bc363fb0bf 100644 --- a/hosting/docker-compose.build.yaml +++ b/hosting/docker-compose.build.yaml @@ -19,7 +19,6 @@ services: API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} JWT_SECRET: ${JWT_SECRET} LOG_LEVEL: info - SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 ENABLE_ANALYTICS: "true" REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} @@ -48,7 +47,6 @@ services: COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 - SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index b3887c15fa..8f66d211f7 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -20,7 +20,6 @@ services: API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} JWT_SECRET: ${JWT_SECRET} LOG_LEVEL: info - SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 ENABLE_ANALYTICS: "true" REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} @@ -31,8 +30,8 @@ services: depends_on: - worker-service - redis-service -# volumes: -# - /some/path/to/plugins:/plugins + # volumes: + # - /some/path/to/plugins:/plugins worker-service: restart: unless-stopped @@ -51,7 +50,6 @@ services: COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 - SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 INTERNAL_API_KEY: ${INTERNAL_API_KEY} REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} @@ -113,7 +111,12 @@ services: PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" depends_on: - couchdb-service - command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] + command: + [ + "sh", + "-c", + "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;", + ] redis-service: restart: unless-stopped diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 95e383edb0..c7b90dbdc4 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -12,14 +12,14 @@ RUN chmod +x /cleanup.sh WORKDIR /app ADD packages/server . COPY yarn.lock . -RUN yarn install --production=true --network-timeout 100000 +RUN yarn install --production=true --network-timeout 1000000 RUN /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . COPY yarn.lock . -RUN yarn install --production=true --network-timeout 100000 +RUN yarn install --production=true --network-timeout 1000000 RUN /cleanup.sh FROM budibase/couchdb diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 new file mode 100644 index 0000000000..b1abe6d53e --- /dev/null +++ b/hosting/single/Dockerfile.v2 @@ -0,0 +1,126 @@ +FROM node:18-slim as build + +# install node-gyp dependencies +RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq + + +# copy and install dependencies +WORKDIR /app +COPY package.json . +COPY yarn.lock . +COPY lerna.json . +COPY .yarnrc . + +COPY packages/server/package.json packages/server/package.json +COPY packages/worker/package.json packages/worker/package.json +# string-templates does not get bundled during the esbuild process, so we want to use the local version +COPY packages/string-templates/package.json packages/string-templates/package.json + + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh + + +# We will never want to sync pro, but the script is still required +RUN echo '' > scripts/syncProPackage.js +RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production + +# copy the actual code +COPY packages/server/dist packages/server/dist +COPY packages/server/pm2.config.js packages/server/pm2.config.js +COPY packages/server/client packages/server/client +COPY packages/server/builder packages/server/builder +COPY packages/worker/dist packages/worker/dist +COPY packages/worker/pm2.config.js packages/worker/pm2.config.js +COPY packages/string-templates packages/string-templates + + +FROM budibase/couchdb as runner +ARG TARGETARCH +ENV TARGETARCH $TARGETARCH +#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) +# e.g. docker build --build-arg TARGETBUILD=aas .... +ARG TARGETBUILD=single +ENV TARGETBUILD $TARGETBUILD + +# install base dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + +# Install postgres client for pg_dump utils +RUN apt install software-properties-common apt-transport-https gpg -y \ + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https gpg -y + +# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx +WORKDIR /nodejs +RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ + bash /tmp/nodesource_setup.sh && \ + apt-get install -y --no-install-recommends libaio1 nodejs && \ + npm install --global yarn pm2 + +# setup nginx +COPY hosting/single/nginx/nginx.conf /etc/nginx +COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default +RUN mkdir -p /var/log/nginx && \ + touch /var/log/nginx/error.log && \ + touch /var/run/nginx.pid && \ + usermod -a -G tty www-data + +WORKDIR / +RUN mkdir -p scripts/integrations/oracle +COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle +RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh + +# setup minio +WORKDIR /minio +COPY scripts/install-minio.sh ./install.sh +RUN chmod +x install.sh && ./install.sh + +# setup runner file +WORKDIR / +COPY hosting/single/runner.sh . +RUN chmod +x ./runner.sh +COPY hosting/single/healthcheck.sh . +RUN chmod +x ./healthcheck.sh + +# Script below sets the path for storing data based on $DATA_DIR +# For Azure App Service install SSH & point data locations to /home +COPY hosting/single/ssh/sshd_config /etc/ +COPY hosting/single/ssh/ssh_setup.sh /tmp +RUN /build-target-paths.sh + + +# setup letsencrypt certificate +RUN apt-get install -y certbot python3-certbot-nginx +COPY hosting/letsencrypt /app/letsencrypt +RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh + +COPY --from=build /app/node_modules /node_modules +COPY --from=build /app/package.json /package.json +COPY --from=build /app/packages/server /app +COPY --from=build /app/packages/worker /worker +COPY --from=build /app/packages/string-templates /string-templates + +RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates + + +EXPOSE 80 +EXPOSE 443 +# Expose port 2222 for SSH on Azure App Service build +EXPOSE 2222 +VOLUME /data + + +HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" + +# must set this just before running +ENV NODE_ENV=production +WORKDIR / + +CMD ["./runner.sh"] diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index d980202f88..9dc7aa25d8 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -7,16 +7,16 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000 +[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app -[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379 +[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1 [[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002 -[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002 -[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001 +[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002 +[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001 [[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app # export CUSTOM_DOMAIN=budi001.custom.com @@ -51,7 +51,7 @@ do fi done if [[ -z "${COUCH_DB_URL}" ]]; then - export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 + export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984 fi if [ ! -f "${DATA_DIR}/.env" ]; then touch ${DATA_DIR}/.env diff --git a/lerna.json b/lerna.json index 0dd6abdc46..9cdf56cbbb 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.34", + "version": "2.11.38", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/nx.json b/nx.json index 8176bae82c..fef6893f9d 100644 --- a/nx.json +++ b/nx.json @@ -8,5 +8,9 @@ } } }, - "targetDefaults": {} + "targetDefaults": { + "build": { + "inputs": ["{workspaceRoot}/scripts/build.js"] + } + } } diff --git a/package.json b/package.json index c38ef76e17..7f5c971009 100644 --- a/package.json +++ b/package.json @@ -3,14 +3,11 @@ "private": true, "devDependencies": { "@esbuild-plugins/tsconfig-paths": "^0.1.2", - "@nx/js": "16.4.3", - "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", "husky": "^8.0.3", - "js-yaml": "^4.1.0", "kill-port": "^1.6.1", "lerna": "7.1.1", "madge": "^6.0.0", @@ -19,8 +16,6 @@ "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", - "rimraf": "^3.0.2", - "rollup-plugin-replace": "^2.2.0", "svelte": "3.49.0", "typescript": "5.2.2", "@babel/core": "^7.22.5", @@ -51,7 +46,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages qa-core --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -61,7 +56,6 @@ "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:pre": "yarn build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", @@ -69,8 +63,7 @@ "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image", + "build:docker:single": "./scripts/build-single-image.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index e9eb77df7c..22ca5b21cc 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -26,7 +26,7 @@ "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", "@techpass/passport-openidconnect": "0.3.2", - "aws-cloudfront-sign": "2.2.0", + "aws-cloudfront-sign": "3.0.2", "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index 420456fd41..bd3efc20db 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -33,8 +33,8 @@ function isInvalid(metadata?: { state: string }) { * Get the requested app metadata by id. * Use redis cache to first read the app metadata. * If not present fallback to loading the app metadata directly and re-caching. - * @param {string} appId the id of the app to get metadata from. - * @returns {object} the app metadata. + * @param appId the id of the app to get metadata from. + * @returns the app metadata. */ export async function getAppMetadata(appId: string): Promise { const client = await getAppClient() @@ -72,9 +72,9 @@ export async function getAppMetadata(appId: string): Promise { /** * Invalidate/reset the cached metadata when a change occurs in the db. - * @param appId {string} the cache key to bust/update. - * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with. - * @return {Promise} will respond with success when cache is updated. + * @param appId the cache key to bust/update. + * @param newMetadata optional - can simply provide the new metadata to update with. + * @return will respond with success when cache is updated. */ export async function invalidateAppMetadata(appId: string, newMetadata?: any) { if (!appId) { diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index 481d3691e4..313b9a4d4a 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -61,9 +61,9 @@ async function populateUsersFromDB( * Get the requested user by id. * Use redis cache to first read the user. * If not present fallback to loading the user directly and re-caching. - * @param {*} userId the id of the user to get - * @param {*} tenantId the tenant of the user to get - * @param {*} populateUser function to provide the user for re-caching. default to couch db + * @param userId the id of the user to get + * @param tenantId the tenant of the user to get + * @param populateUser function to provide the user for re-caching. default to couch db * @returns */ export async function getUser( @@ -111,8 +111,8 @@ export async function getUser( * Get the requested users by id. * Use redis cache to first read the users. * If not present fallback to loading the users directly and re-caching. - * @param {*} userIds the ids of the user to get - * @param {*} tenantId the tenant of the users to get + * @param userIds the ids of the user to get + * @param tenantId the tenant of the users to get * @returns */ export async function getUsers( diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index e64c116663..c331d791a6 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -119,8 +119,8 @@ export class Writethrough { this.writeRateMs = writeRateMs } - async put(doc: any) { - return put(this.db, doc, this.writeRateMs) + async put(doc: any, writeRateMs: number = this.writeRateMs) { + return put(this.db, doc, writeRateMs) } async get(id: string) { diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 49ace84d52..0c83ed005d 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -23,7 +23,7 @@ import environment from "../environment" /** * Generates a new configuration ID. - * @returns {string} The new configuration ID which the config doc can be stored under. + * @returns The new configuration ID which the config doc can be stored under. */ export function generateConfigID(type: ConfigType) { return `${DocumentType.CONFIG}${SEPARATOR}${type}` diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 61d96bb4b0..609c18abb5 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -62,7 +62,7 @@ export function isTenancyEnabled() { /** * Given an app ID this will attempt to retrieve the tenant ID from it. - * @return {null|string} The tenant ID found within the app ID. + * @return The tenant ID found within the app ID. */ export function getTenantIDFromAppID(appId: string) { if (!appId) { diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index e813722d98..f91a37ce8f 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -8,8 +8,8 @@ class Replication { /** * - * @param {String} source - the DB you want to replicate or rollback to - * @param {String} target - the DB you want to replicate to, or rollback from + * @param source - the DB you want to replicate or rollback to + * @param target - the DB you want to replicate to, or rollback from */ constructor({ source, target }: any) { this.source = getPouchDB(source) @@ -38,7 +38,7 @@ class Replication { /** * Two way replication operation, intended to be promise based. - * @param {Object} opts - PouchDB replication options + * @param opts - PouchDB replication options */ sync(opts = {}) { this.replication = this.promisify(this.source.sync, opts) @@ -47,7 +47,7 @@ class Replication { /** * One way replication operation, intended to be promise based. - * @param {Object} opts - PouchDB replication options + * @param opts - PouchDB replication options */ replicate(opts = {}) { this.replication = this.promisify(this.source.replicate.to, opts) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 7451d581b5..f982ee67d0 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -599,10 +599,10 @@ async function runQuery( * Gets round the fixed limit of 200 results from a query by fetching as many * pages as required and concatenating the results. This recursively operates * until enough results have been found. - * @param dbName {string} Which database to run a lucene query on - * @param index {string} Which search index to utilise - * @param query {object} The JSON query structure - * @param params {object} The search params including: + * @param dbName Which database to run a lucene query on + * @param index Which search index to utilise + * @param query The JSON query structure + * @param params The search params including: * tableId {string} The table ID to search * sort {string} The sort column * sortOrder {string} The sort order ("ascending" or "descending") @@ -655,10 +655,10 @@ async function recursiveSearch( * Performs a paginated search. A bookmark will be returned to allow the next * page to be fetched. There is a max limit off 200 results per page in a * paginated search. - * @param dbName {string} Which database to run a lucene query on - * @param index {string} Which search index to utilise - * @param query {object} The JSON query structure - * @param params {object} The search params including: + * @param dbName Which database to run a lucene query on + * @param index Which search index to utilise + * @param query The JSON query structure + * @param params The search params including: * tableId {string} The table ID to search * sort {string} The sort column * sortOrder {string} The sort order ("ascending" or "descending") @@ -722,10 +722,10 @@ export async function paginatedSearch( * desired amount of results. There is a limit of 1000 results to avoid * heavy performance hits, and to avoid client components breaking from * handling too much data. - * @param dbName {string} Which database to run a lucene query on - * @param index {string} Which search index to utilise - * @param query {object} The JSON query structure - * @param params {object} The search params including: + * @param dbName Which database to run a lucene query on + * @param index Which search index to utilise + * @param query The JSON query structure + * @param params The search params including: * tableId {string} The table ID to search * sort {string} The sort column * sortOrder {string} The sort order ("ascending" or "descending") diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 4ebf8392b5..d7a4b8224a 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -45,7 +45,7 @@ export async function getAllDbs(opts = { efficient: false }) { * Lots of different points in the system need to find the full list of apps, this will * enumerate the entire CouchDB cluster and get the list of databases (every app). * - * @return {Promise} returns the app information document stored in each app database. + * @return returns the app information document stored in each app database. */ export async function getAllApps({ dev, diff --git a/packages/backend-core/src/docIds/conversions.ts b/packages/backend-core/src/docIds/conversions.ts index 381c5cb90f..b168b74e16 100644 --- a/packages/backend-core/src/docIds/conversions.ts +++ b/packages/backend-core/src/docIds/conversions.ts @@ -25,7 +25,7 @@ export function isDevApp(app: App) { /** * Generates a development app ID from a real app ID. - * @returns {string} the dev app ID which can be used for dev database. + * @returns the dev app ID which can be used for dev database. */ export function getDevelopmentAppID(appId: string) { if (!appId || appId.startsWith(APP_DEV_PREFIX)) { diff --git a/packages/backend-core/src/docIds/ids.ts b/packages/backend-core/src/docIds/ids.ts index 4c9eb713c8..02176109da 100644 --- a/packages/backend-core/src/docIds/ids.ts +++ b/packages/backend-core/src/docIds/ids.ts @@ -8,7 +8,7 @@ import { newid } from "./newid" /** * Generates a new app ID. - * @returns {string} The new app ID which the app doc can be stored under. + * @returns The new app ID which the app doc can be stored under. */ export const generateAppID = (tenantId?: string | null) => { let id = APP_PREFIX @@ -20,9 +20,9 @@ export const generateAppID = (tenantId?: string | null) => { /** * Gets a new row ID for the specified table. - * @param {string} tableId The table which the row is being created for. - * @param {string|null} id If an ID is to be used then the UUID can be substituted for this. - * @returns {string} The new ID which a row doc can be stored under. + * @param tableId The table which the row is being created for. + * @param id If an ID is to be used then the UUID can be substituted for this. + * @returns The new ID which a row doc can be stored under. */ export function generateRowID(tableId: string, id?: string) { id = id || newid() @@ -31,7 +31,7 @@ export function generateRowID(tableId: string, id?: string) { /** * Generates a new workspace ID. - * @returns {string} The new workspace ID which the workspace doc can be stored under. + * @returns The new workspace ID which the workspace doc can be stored under. */ export function generateWorkspaceID() { return `${DocumentType.WORKSPACE}${SEPARATOR}${newid()}` @@ -39,7 +39,7 @@ export function generateWorkspaceID() { /** * Generates a new global user ID. - * @returns {string} The new user ID which the user doc can be stored under. + * @returns The new user ID which the user doc can be stored under. */ export function generateGlobalUserID(id?: any) { return `${DocumentType.USER}${SEPARATOR}${id || newid()}` @@ -52,8 +52,8 @@ export function isGlobalUserID(id: string) { /** * Generates a new user ID based on the passed in global ID. - * @param {string} globalId The ID of the global user. - * @returns {string} The new user ID which the user doc can be stored under. + * @param globalId The ID of the global user. + * @returns The new user ID which the user doc can be stored under. */ export function generateUserMetadataID(globalId: string) { return generateRowID(InternalTable.USER_METADATA, globalId) @@ -84,7 +84,7 @@ export function generateAppUserID(prodAppId: string, userId: string) { /** * Generates a new role ID. - * @returns {string} The new role ID which the role doc can be stored under. + * @returns The new role ID which the role doc can be stored under. */ export function generateRoleID(name: string) { const prefix = `${DocumentType.ROLE}${SEPARATOR}` @@ -103,7 +103,7 @@ export function prefixRoleID(name: string) { /** * Generates a new dev info document ID - this is scoped to a user. - * @returns {string} The new dev info ID which info for dev (like api key) can be stored under. + * @returns The new dev info ID which info for dev (like api key) can be stored under. */ export const generateDevInfoID = (userId: any) => { return `${DocumentType.DEV_INFO}${SEPARATOR}${userId}` @@ -111,7 +111,7 @@ export const generateDevInfoID = (userId: any) => { /** * Generates a new plugin ID - to be used in the global DB. - * @returns {string} The new plugin ID which a plugin metadata document can be stored under. + * @returns The new plugin ID which a plugin metadata document can be stored under. */ export const generatePluginID = (name: string) => { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 5d563952f7..36fd75622b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -12,12 +12,12 @@ import { getProdAppID } from "./conversions" * is usually the case as most of our docs are top level e.g. tables, automations, users and so on. * More complex cases such as link docs and rows which have multiple levels of IDs that their * ID consists of need their own functions to build the allDocs parameters. - * @param {string} docType The type of document which input params are being built for, e.g. user, + * @param docType The type of document which input params are being built for, e.g. user, * link, app, table and so on. - * @param {string|null} docId The ID of the document minus its type - this is only needed if looking + * @param docId The ID of the document minus its type - this is only needed if looking * for a singular document. - * @param {object} otherProps Add any other properties onto the request, e.g. include_docs. - * @returns {object} Parameters which can then be used with an allDocs request. + * @param otherProps Add any other properties onto the request, e.g. include_docs. + * @returns Parameters which can then be used with an allDocs request. */ export function getDocParams( docType: string, @@ -36,11 +36,11 @@ export function getDocParams( /** * Gets the DB allDocs/query params for retrieving a row. - * @param {string|null} tableId The table in which the rows have been stored. - * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be + * @param tableId The table in which the rows have been stored. + * @param rowId The ID of the row which is being specifically queried for. This can be * left null to get all the rows in the table. - * @param {object} otherProps Any other properties to add to the request. - * @returns {object} Parameters which can then be used with an allDocs request. + * @param otherProps Any other properties to add to the request. + * @returns Parameters which can then be used with an allDocs request. */ export function getRowParams( tableId?: string | null, diff --git a/packages/backend-core/src/helpers.ts b/packages/backend-core/src/helpers.ts index e1e065bd4e..dd241f4af7 100644 --- a/packages/backend-core/src/helpers.ts +++ b/packages/backend-core/src/helpers.ts @@ -1,8 +1,8 @@ /** * Makes sure that a URL has the correct number of slashes, while maintaining the * http(s):// double slashes. - * @param {string} url The URL to test and remove any extra double slashes. - * @return {string} The updated url. + * @param url The URL to test and remove any extra double slashes. + * @return The updated url. */ export function checkSlashesInUrl(url: string) { return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2") diff --git a/packages/backend-core/src/middleware/passport/local.ts b/packages/backend-core/src/middleware/passport/local.ts index e198032532..f1d72cab7a 100644 --- a/packages/backend-core/src/middleware/passport/local.ts +++ b/packages/backend-core/src/middleware/passport/local.ts @@ -13,10 +13,10 @@ export const options = { /** * Passport Local Authentication Middleware. - * @param {*} ctx the request structure - * @param {*} email username to login with - * @param {*} password plain text password to log in with - * @param {*} done callback from passport to return user information and errors + * @param ctx the request structure + * @param email username to login with + * @param password plain text password to log in with + * @param done callback from passport to return user information and errors * @returns The authenticated user, or errors if they occur */ export async function authenticate( diff --git a/packages/backend-core/src/middleware/passport/sso/oidc.ts b/packages/backend-core/src/middleware/passport/sso/oidc.ts index 83bfde28b6..061e0507aa 100644 --- a/packages/backend-core/src/middleware/passport/sso/oidc.ts +++ b/packages/backend-core/src/middleware/passport/sso/oidc.ts @@ -17,15 +17,15 @@ const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) { /** - * @param {*} issuer The identity provider base URL - * @param {*} sub The user ID - * @param {*} profile The user profile information. Created by passport from the /userinfo response - * @param {*} jwtClaims The parsed id_token claims - * @param {*} accessToken The access_token for contacting the identity provider - may or may not be a JWT - * @param {*} refreshToken The refresh_token for obtaining a new access_token - usually not a JWT - * @param {*} idToken The id_token - always a JWT - * @param {*} params The response body from requesting an access_token - * @param {*} done The passport callback: err, user, info + * @param issuer The identity provider base URL + * @param sub The user ID + * @param profile The user profile information. Created by passport from the /userinfo response + * @param jwtClaims The parsed id_token claims + * @param accessToken The access_token for contacting the identity provider - may or may not be a JWT + * @param refreshToken The refresh_token for obtaining a new access_token - usually not a JWT + * @param idToken The id_token - always a JWT + * @param params The response body from requesting an access_token + * @param done The passport callback: err, user, info */ return async ( issuer: string, @@ -61,8 +61,8 @@ export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) { } /** - * @param {*} profile The structured profile created by passport using the user info endpoint - * @param {*} jwtClaims The claims returned in the id token + * @param profile The structured profile created by passport using the user info endpoint + * @param jwtClaims The claims returned in the id token */ function getEmail(profile: SSOProfile, jwtClaims: JwtClaims) { // profile not guaranteed to contain email e.g. github connected azure ad account diff --git a/packages/backend-core/src/middleware/passport/utils.ts b/packages/backend-core/src/middleware/passport/utils.ts index 7e0d3863a0..88642471b9 100644 --- a/packages/backend-core/src/middleware/passport/utils.ts +++ b/packages/backend-core/src/middleware/passport/utils.ts @@ -5,9 +5,9 @@ import { ConfigType, GoogleInnerConfig } from "@budibase/types" /** * Utility to handle authentication errors. * - * @param {*} done The passport callback. - * @param {*} message Message that will be returned in the response body - * @param {*} err (Optional) error that will be logged + * @param done The passport callback. + * @param message Message that will be returned in the response body + * @param err (Optional) error that will be logged */ export function authError(done: Function, message: string, err?: any) { diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index 9951058d6a..be9fddeaa6 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -6,10 +6,10 @@ import * as cloudfront from "../cloudfront" * In production the client library is stored in the object store, however in development * we use the symlinked version produced by lerna, located in node modules. We link to this * via a specific endpoint (under /api/assets/client). - * @param {string} appId In production we need the appId to look up the correct bucket, as the + * @param appId In production we need the appId to look up the correct bucket, as the * version of the client lib may differ between apps. - * @param {string} version The version to retrieve. - * @return {string} The URL to be inserted into appPackage response or server rendered + * @param version The version to retrieve. + * @return The URL to be inserted into appPackage response or server rendered * app index file. */ export const clientLibraryUrl = (appId: string, version: string) => { diff --git a/packages/backend-core/src/objectStore/cloudfront.ts b/packages/backend-core/src/objectStore/cloudfront.ts index a61ea7f583..866fe9e880 100644 --- a/packages/backend-core/src/objectStore/cloudfront.ts +++ b/packages/backend-core/src/objectStore/cloudfront.ts @@ -1,5 +1,5 @@ import env from "../environment" -const cfsign = require("aws-cloudfront-sign") +import * as cfsign from "aws-cloudfront-sign" let PRIVATE_KEY: string | undefined @@ -21,7 +21,7 @@ function getPrivateKey() { const getCloudfrontSignParams = () => { return { - keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID, + keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!, privateKeyString: getPrivateKey(), expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 4ac3641de1..c36a09915e 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -61,9 +61,9 @@ export function sanitizeBucket(input: string) { /** * Gets a connection to the object store using the S3 SDK. - * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from. - * @param {object} opts configuration for the object store. - * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage. + * @param bucket the name of the bucket which blobs will be uploaded/retrieved from. + * @param opts configuration for the object store. + * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ export const ObjectStore = ( diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index f73ded0659..8974a9f5a2 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -6,6 +6,7 @@ import { AutomationStepIdArray, AutomationIOType, AutomationCustomIOType, + DatasourceFeature, } from "@budibase/types" import joi from "joi" @@ -67,9 +68,27 @@ function validateDatasource(schema: any) { version: joi.string().optional(), schema: joi.object({ docs: joi.string(), + plus: joi.boolean().optional(), + isSQL: joi.boolean().optional(), + auth: joi + .object({ + type: joi.string().required(), + }) + .optional(), + features: joi + .object( + Object.fromEntries( + Object.values(DatasourceFeature).map(key => [ + key, + joi.boolean().optional(), + ]) + ) + ) + .optional(), + relationships: joi.boolean().optional(), + description: joi.string().required(), friendlyName: joi.string().required(), type: joi.string().allow(...DATASOURCE_TYPES), - description: joi.string().required(), datasource: joi.object().pattern(joi.string(), fieldValidator).required(), query: joi .object() diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index ec1d9d4a90..af2ec6dbaa 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -5,9 +5,9 @@ import { timeout } from "../utils" * Bull works with a Job wrapper around all messages that contains a lot more information about * the state of the message, this object constructor implements the same schema of Bull jobs * for the sake of maintaining API consistency. - * @param {string} queue The name of the queue which the message will be carried on. - * @param {object} message The JSON message which will be passed back to the consumer. - * @returns {Object} A new job which can now be put onto the queue, this is mostly an + * @param queue The name of the queue which the message will be carried on. + * @param message The JSON message which will be passed back to the consumer. + * @returns A new job which can now be put onto the queue, this is mostly an * internal structure so that an in memory queue can be easily swapped for a Bull queue. */ function newJob(queue: string, message: any) { @@ -32,8 +32,8 @@ class InMemoryQueue { _addCount: number /** * The constructor the queue, exactly the same as that of Bulls. - * @param {string} name The name of the queue which is being configured. - * @param {object|null} opts This is not used by the in memory queue as there is no real use + * @param name The name of the queue which is being configured. + * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ constructor(name: string, opts = null) { @@ -49,7 +49,7 @@ class InMemoryQueue { * Same callback API as Bull, each callback passed to this will consume messages as they are * available. Please note this is a queue service, not a notification service, so each * consumer will receive different messages. - * @param {function} func The callback function which will return a "Job", the same + * @param func The callback function which will return a "Job", the same * as the Bull API, within this job the property "data" contains the JSON message. Please * note this is incredibly limited compared to Bull as in reality the Job would contain * a lot more information about the queue and current status of Bull cluster. @@ -73,9 +73,9 @@ class InMemoryQueue { * Simple function to replicate the add message functionality of Bull, putting * a new message on the queue. This then emits an event which will be used to * return the message to a consumer (if one is attached). - * @param {object} msg A message to be transported over the queue, this should be + * @param msg A message to be transported over the queue, this should be * a JSON message as this is required by Bull. - * @param {boolean} repeat serves no purpose for the import queue. + * @param repeat serves no purpose for the import queue. */ // eslint-disable-next-line no-unused-vars add(msg: any, repeat: boolean) { @@ -96,7 +96,7 @@ class InMemoryQueue { /** * This removes a cron which has been implemented, this is part of Bull API. - * @param {string} cronJobId The cron which is to be removed. + * @param cronJobId The cron which is to be removed. */ removeRepeatableByKey(cronJobId: string) { // TODO: implement for testing diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index e7755f275d..d1e2d8989e 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -142,7 +142,7 @@ function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) { * this can only be done with redis streams because they will have an end. * @param stream A redis stream, specifically as this type of stream will have an end. * @param client The client to use for further lookups. - * @return {Promise} The final output of the stream + * @return The final output of the stream */ function promisifyStream(stream: any, client: RedisWrapper) { return new Promise((resolve, reject) => { diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 539bbaef27..fe4095d210 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -36,8 +36,8 @@ export function levelToNumber(perm: PermissionLevel) { /** * Given the specified permission level for the user return the levels they are allowed to carry out. - * @param {string} userPermLevel The permission level of the user. - * @return {string[]} All the permission levels this user is allowed to carry out. + * @param userPermLevel The permission level of the user. + * @return All the permission levels this user is allowed to carry out. */ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] { switch (userPermLevel) { diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 24279e6b5c..b05cf79c8c 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -149,9 +149,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { /** * Gets the role object, this is mainly useful for two purposes, to check if the level exists and * to check if the role inherits any others. - * @param {string|null} roleId The level ID to lookup. - * @param {object|null} opts options for the function, like whether to halt errors, instead return public. - * @returns {Promise} The role object, which may contain an "inherits" property. + * @param roleId The level ID to lookup. + * @param opts options for the function, like whether to halt errors, instead return public. + * @returns The role object, which may contain an "inherits" property. */ export async function getRole( roleId?: string, @@ -225,8 +225,8 @@ export async function getUserRoleIdHierarchy( /** * Returns an ordered array of the user's inherited role IDs, this can be used * to determine if a user can access something that requires a specific role. - * @param {string} userRoleId The user's role ID, this can be found in their access token. - * @returns {Promise} returns an ordered array of the roles, with the first being their + * @param userRoleId The user's role ID, this can be found in their access token. + * @returns returns an ordered array of the roles, with the first being their * highest level of access and the last being the lowest level. */ export async function getUserRoleHierarchy(userRoleId?: string) { @@ -258,7 +258,7 @@ export async function getAllRoleIds(appId?: string) { /** * Given an app ID this will retrieve all of the roles that are currently within that app. - * @return {Promise} An array of the role objects that were found. + * @return An array of the role objects that were found. */ export async function getAllRoles(appId?: string): Promise { if (appId) { diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 1d02bebc32..8bb6300d4e 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -21,17 +21,21 @@ import { User, UserStatus, UserGroup, - ContextUser, } from "@budibase/types" import { getAccountHolderFromUserIds, isAdmin, + isCreator, validateUniqueUser, } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" -type QuotaUpdateFn = (change: number, cb?: () => Promise) => Promise +type QuotaUpdateFn = ( + change: number, + creatorsChange: number, + cb?: () => Promise +) => Promise type GroupUpdateFn = (groupId: string, userIds: string[]) => Promise type FeatureFn = () => Promise type GroupGetFn = (ids: string[]) => Promise @@ -135,7 +139,7 @@ export class UserDB { if (!fullUser.roles) { fullUser.roles = {} } - // add the active status to a user if its not provided + // add the active status to a user if it's not provided if (fullUser.status == null) { fullUser.status = UserStatus.ACTIVE } @@ -246,7 +250,8 @@ export class UserDB { } const change = dbUser ? 0 : 1 // no change if there is existing user - return UserDB.quotas.addUsers(change, async () => { + const creatorsChange = isCreator(dbUser) !== isCreator(user) ? 1 : 0 + return UserDB.quotas.addUsers(change, creatorsChange, async () => { await validateUniqueUser(email, tenantId) let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser) @@ -308,6 +313,7 @@ export class UserDB { let usersToSave: any[] = [] let newUsers: any[] = [] + let newCreators: any[] = [] const emails = newUsersRequested.map((user: User) => user.email) const existingEmails = await searchExistingEmails(emails) @@ -328,59 +334,66 @@ export class UserDB { } newUser.userGroups = groups newUsers.push(newUser) + if (isCreator(newUser)) { + newCreators.push(newUser) + } } const account = await accountSdk.getAccountByTenantId(tenantId) - return UserDB.quotas.addUsers(newUsers.length, async () => { - // create the promises array that will be called by bulkDocs - newUsers.forEach((user: any) => { - usersToSave.push( - UserDB.buildUser( - user, - { - hashPassword: true, - requirePassword: user.requirePassword, - }, - tenantId, - undefined, // no dbUser - account + return UserDB.quotas.addUsers( + newUsers.length, + newCreators.length, + async () => { + // create the promises array that will be called by bulkDocs + newUsers.forEach((user: any) => { + usersToSave.push( + UserDB.buildUser( + user, + { + hashPassword: true, + requirePassword: user.requirePassword, + }, + tenantId, + undefined, // no dbUser + account + ) ) - ) - }) + }) - const usersToBulkSave = await Promise.all(usersToSave) - await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) + const usersToBulkSave = await Promise.all(usersToSave) + await usersCore.bulkUpdateGlobalUsers(usersToBulkSave) - // Post-processing of bulk added users, e.g. events and cache operations - for (const user of usersToBulkSave) { - // TODO: Refactor to bulk insert users into the info db - // instead of relying on looping tenant creation - await platform.users.addUser(tenantId, user._id, user.email) - await eventHelpers.handleSaveEvents(user, undefined) - } + // Post-processing of bulk added users, e.g. events and cache operations + for (const user of usersToBulkSave) { + // TODO: Refactor to bulk insert users into the info db + // instead of relying on looping tenant creation + await platform.users.addUser(tenantId, user._id, user.email) + await eventHelpers.handleSaveEvents(user, undefined) + } + + const saved = usersToBulkSave.map(user => { + return { + _id: user._id, + email: user.email, + } + }) + + // now update the groups + if (Array.isArray(saved) && groups) { + const groupPromises = [] + const createdUserIds = saved.map(user => user._id) + for (let groupId of groups) { + groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) + } + await Promise.all(groupPromises) + } - const saved = usersToBulkSave.map(user => { return { - _id: user._id, - email: user.email, + successful: saved, + unsuccessful, } - }) - - // now update the groups - if (Array.isArray(saved) && groups) { - const groupPromises = [] - const createdUserIds = saved.map(user => user._id) - for (let groupId of groups) { - groupPromises.push(UserDB.groups.addUsers(groupId, createdUserIds)) - } - await Promise.all(groupPromises) } - - return { - successful: saved, - unsuccessful, - } - }) + ) } static async bulkDelete(userIds: string[]): Promise { @@ -420,11 +433,12 @@ export class UserDB { _deleted: true, })) const dbResponse = await usersCore.bulkUpdateGlobalUsers(toDelete) + const creatorsToDelete = usersToDelete.filter(isCreator) - await UserDB.quotas.removeUsers(toDelete.length) for (let user of usersToDelete) { await bulkDeleteProcessing(user) } + await UserDB.quotas.removeUsers(toDelete.length, creatorsToDelete.length) // Build Response // index users by id @@ -473,7 +487,8 @@ export class UserDB { await db.remove(userId, dbUser._rev) - await UserDB.quotas.removeUsers(1) + const creatorsToDelete = isCreator(dbUser) ? 1 : 0 + await UserDB.quotas.removeUsers(1, creatorsToDelete) await eventHelpers.handleDeleteEvents(dbUser) await cache.user.invalidateUser(userId) await sessions.invalidateSessions(userId, { reason: "deletion" }) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index b087a6b538..a64997224e 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,14 +14,15 @@ import { } from "../db" import { BulkDocsResponse, - ContextUser, SearchQuery, SearchQueryOperators, SearchUsersRequest, User, + ContextUser, } from "@budibase/types" -import * as context from "../context" import { getGlobalDB } from "../context" +import * as context from "../context" +import { isCreator } from "./utils" type GetOpts = { cleanup?: boolean } @@ -283,6 +284,19 @@ export async function getUserCount() { return response.total_rows } +export async function getCreatorCount() { + let creators = 0 + async function iterate(startPage?: string) { + const page = await paginatedUsers({ bookmark: startPage }) + creators += page.data.filter(isCreator).length + if (page.hasNextPage) { + await iterate(page.nextPage) + } + } + await iterate() + return creators +} + // used to remove the builder/admin permissions, for processing the // user as an app user (they may have some specific role/group export function removePortalUserPermissions(user: User | ContextUser) { diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index af0e8e10c7..0ef4b77998 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -10,6 +10,7 @@ import { getAccountByTenantId } from "../accounts" // extract from shared-core to make easily accessible from backend-core export const isBuilder = sdk.users.isBuilder export const isAdmin = sdk.users.isAdmin +export const isCreator = sdk.users.isCreator export const isGlobalBuilder = sdk.users.isGlobalBuilder export const isAdminOrBuilder = sdk.users.isAdminOrBuilder export const hasAdminPermissions = sdk.users.hasAdminPermissions diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ac43fa1fdb..b92471a7a4 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -79,8 +79,8 @@ export function isPublicApiRequest(ctx: Ctx): boolean { /** * Given a request tries to find the appId, which can be located in various places - * @param {object} ctx The main request body to look through. - * @returns {string|undefined} If an appId was found it will be returned. + * @param ctx The main request body to look through. + * @returns If an appId was found it will be returned. */ export async function getAppIdFromCtx(ctx: Ctx) { // look in headers @@ -135,7 +135,7 @@ function parseAppIdFromUrl(url?: string) { /** * opens the contents of the specified encrypted JWT. - * @return {object} the contents of the token. + * @return the contents of the token. */ export function openJwt(token: string) { if (!token) { @@ -169,8 +169,8 @@ export function isValidInternalAPIKey(apiKey: string) { /** * Get a cookie from context, and decrypt if necessary. - * @param {object} ctx The request which is to be manipulated. - * @param {string} name The name of the cookie to get. + * @param ctx The request which is to be manipulated. + * @param name The name of the cookie to get. */ export function getCookie(ctx: Ctx, name: string) { const cookie = ctx.cookies.get(name) @@ -184,10 +184,10 @@ export function getCookie(ctx: Ctx, name: string) { /** * Store a cookie for the request - it will not expire. - * @param {object} ctx The request which is to be manipulated. - * @param {string} name The name of the cookie to set. - * @param {string|object} value The value of cookie which will be set. - * @param {object} opts options like whether to sign. + * @param ctx The request which is to be manipulated. + * @param name The name of the cookie to set. + * @param value The value of cookie which will be set. + * @param opts options like whether to sign. */ export function setCookie( ctx: Ctx, @@ -223,8 +223,8 @@ export function clearCookie(ctx: Ctx, name: string) { /** * Checks if the API call being made (based on the provided ctx object) is from the client. If * the call is not from a client app then it is from the builder. - * @param {object} ctx The koa context object to be tested. - * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder). + * @param ctx The koa context object to be tested. + * @return returns true if the call is from the client lib (a built app rather than the builder). */ export function isClient(ctx: Ctx) { return ctx.headers[Header.TYPE] === "client" diff --git a/packages/backend-core/tests/core/users/users.spec.js b/packages/backend-core/tests/core/users/users.spec.js new file mode 100644 index 0000000000..ae7109344a --- /dev/null +++ b/packages/backend-core/tests/core/users/users.spec.js @@ -0,0 +1,54 @@ +const _ = require('lodash/fp') +const {structures} = require("../../../tests") + +jest.mock("../../../src/context") +jest.mock("../../../src/db") + +const context = require("../../../src/context") +const db = require("../../../src/db") + +const {getCreatorCount} = require('../../../src/users/users') + +describe("Users", () => { + + let getGlobalDBMock + let getGlobalUserParamsMock + let paginationMock + + beforeEach(() => { + jest.resetAllMocks() + + getGlobalDBMock = jest.spyOn(context, "getGlobalDB") + getGlobalUserParamsMock = jest.spyOn(db, "getGlobalUserParams") + paginationMock = jest.spyOn(db, "pagination") + }) + + it("Retrieves the number of creators", async () => { + const getUsers = (offset, limit, creators = false) => { + const range = _.range(offset, limit) + const opts = creators ? {builder: {global: true}} : undefined + return range.map(() => structures.users.user(opts)) + } + const page1Data = getUsers(0, 8) + const page2Data = getUsers(8, 12, true) + getGlobalDBMock.mockImplementation(() => ({ + name : "fake-db", + allDocs: () => ({ + rows: [...page1Data, ...page2Data] + }) + })) + paginationMock.mockImplementationOnce(() => ({ + data: page1Data, + hasNextPage: true, + nextPage: "1" + })) + paginationMock.mockImplementation(() => ({ + data: page2Data, + hasNextPage: false, + nextPage: undefined + })) + const creatorsCount = await getCreatorCount() + expect(creatorsCount).toBe(4) + expect(paginationMock).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 5cce84edfd..bb452f9ad5 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -72,6 +72,11 @@ export function quotas(): Quotas { value: 1, triggers: [], }, + creators: { + name: "Creators", + value: 1, + triggers: [], + }, userGroups: { name: "User Groups", value: 1, @@ -118,6 +123,10 @@ export function customer(): Customer { export function subscription(): Subscription { return { amount: 10000, + amounts: { + user: 10000, + creator: 0, + }, cancelAt: undefined, currency: "usd", currentPeriodEnd: 0, @@ -126,6 +135,10 @@ export function subscription(): Subscription { duration: PriceDuration.MONTHLY, pastDueAt: undefined, quantity: 0, + quantities: { + user: 0, + creator: 0, + }, status: "active", } } diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts index e82117053f..8d0b05fe1e 100644 --- a/packages/backend-core/tests/core/utilities/structures/quotas.ts +++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts @@ -1,6 +1,6 @@ import { MonthlyQuotaName, QuotaUsage } from "@budibase/types" -export const usage = (): QuotaUsage => { +export const usage = (users: number = 0, creators: number = 0): QuotaUsage => { return { _id: "usage_quota", quotaReset: new Date().toISOString(), @@ -58,7 +58,8 @@ export const usage = (): QuotaUsage => { usageQuota: { apps: 0, plugins: 0, - users: 0, + users, + creators, userGroups: 0, rows: 0, triggers: {}, diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 4791776c57..78eed2b608 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -82,9 +82,9 @@ "@spectrum-css/vars": "3.0.1", "dayjs": "^1.10.8", "easymde": "^2.16.1", + "svelte-dnd-action": "^0.9.8", "svelte-flatpickr": "3.2.3", - "svelte-portal": "^1.0.0", - "svelte-dnd-action": "^0.9.8" + "svelte-portal": "^1.0.0" }, "resolutions": { "loader-utils": "1.4.1" diff --git a/packages/builder/build/copy.js b/packages/builder/build/copy.js deleted file mode 100644 index f3077b7854..0000000000 --- a/packages/builder/build/copy.js +++ /dev/null @@ -1,8 +0,0 @@ -const ncp = require("ncp").ncp - -ncp("./dist", "../server/builder", function (err) { - if (err) { - return console.error(err) - } - console.log("Copied dist folder to ../server/builder") -}) diff --git a/packages/builder/package.json b/packages/builder/package.json index 20530cad20..3cc5612652 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -64,7 +64,6 @@ "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", - "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", "codemirror": "^5.59.0", @@ -85,8 +84,8 @@ "@babel/core": "^7.12.14", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", - "@rollup/plugin-replace": "^2.4.2", - "@roxi/routify": "2.18.5", + "@rollup/plugin-replace": "^5.0.3", + "@roxi/routify": "2.18.12", "@sveltejs/vite-plugin-svelte": "1.0.1", "@testing-library/jest-dom": "5.17.0", "@testing-library/svelte": "^3.2.2", @@ -95,16 +94,18 @@ "jest": "29.6.2", "jsdom": "^21.1.1", "ncp": "^2.0.0", - "rollup": "^2.44.0", "svelte": "^3.48.0", "svelte-jester": "^1.3.2", - "vite": "^3.0.8", - "vite-plugin-static-copy": "^0.16.0", + "vite": "^4.4.11", + "vite-plugin-static-copy": "^0.17.0", "vitest": "^0.29.2" }, "nx": { "targets": { "build": { + "outputs": [ + "{workspaceRoot}/packages/server/builder" + ], "dependsOn": [ { "projects": [ diff --git a/packages/builder/src/analytics/SentryClient.js b/packages/builder/src/analytics/SentryClient.js deleted file mode 100644 index 2a1f8732e3..0000000000 --- a/packages/builder/src/analytics/SentryClient.js +++ /dev/null @@ -1,37 +0,0 @@ -import * as Sentry from "@sentry/browser" - -export default class SentryClient { - constructor(dsn) { - this.dsn = dsn - } - - init() { - if (this.dsn) { - Sentry.init({ dsn: this.dsn }) - - this.initalised = true - } - } - - /** - * Capture an exception and send it to sentry. - * @param {Error} err - JS error object - */ - captureException(err) { - if (!this.initalised) return - - Sentry.captureException(err) - } - - /** - * Identify user in sentry. - * @param {String} id - Unique user id - */ - identify(id) { - if (!this.initalised) return - - Sentry.configureScope(scope => { - scope.setUser({ id }) - }) - } -} diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index e49ec6d197..6bb10acdb5 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -1,16 +1,14 @@ import { API } from "api" import PosthogClient from "./PosthogClient" import IntercomClient from "./IntercomClient" -import SentryClient from "./SentryClient" import { Events, EventSource } from "./constants" const posthog = new PosthogClient(process.env.POSTHOG_TOKEN) -const sentry = new SentryClient(process.env.SENTRY_DSN) const intercom = new IntercomClient(process.env.INTERCOM_TOKEN) class AnalyticsHub { constructor() { - this.clients = [posthog, sentry, intercom] + this.clients = [posthog, intercom] } async activate() { @@ -23,12 +21,9 @@ class AnalyticsHub { identify(id) { posthog.identify(id) - sentry.identify(id) } - captureException(err) { - sentry.captureException(err) - } + captureException(_err) {} captureEvent(eventName, props = {}) { posthog.captureEvent(eventName, props) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ba61ede746..7b51e6c839 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -36,7 +36,7 @@ import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" - const AUTO_TYPE = "auto" + const AUTO_TYPE = FIELDS.AUTO.type const FORMULA_TYPE = FIELDS.FORMULA.type const LINK_TYPE = FIELDS.LINK.type const STRING_TYPE = FIELDS.STRING.type @@ -60,8 +60,13 @@ {} ) - function makeFieldId(type, subtype) { - return `${type}${subtype || ""}`.toUpperCase() + function makeFieldId(type, subtype, autocolumn) { + // don't make field IDs for auto types + if (type === AUTO_TYPE || autocolumn) { + return type.toUpperCase() + } else { + return `${type}${subtype || ""}`.toUpperCase() + } } let originalName @@ -183,7 +188,8 @@ if (!savingColumn) { editableColumn.fieldId = makeFieldId( editableColumn.type, - editableColumn.subtype + editableColumn.subtype, + editableColumn.autocolumn ) allowedTypes = getAllowedTypes().map(t => ({ @@ -419,7 +425,7 @@ FIELDS.FORMULA, FIELDS.JSON, isUsers ? FIELDS.USERS : FIELDS.USER, - { name: "Auto Column", type: AUTO_TYPE }, + FIELDS.AUTO, ] } else { let fields = [ @@ -538,7 +544,7 @@ getOptionValue={field => field.fieldId} getOptionIcon={field => field.icon} isOptionEnabled={option => { - if (option.type == AUTO_TYPE) { + if (option.type === AUTO_TYPE) { return availableAutoColumnKeys?.length > 0 } return true diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 9e42dfecd9..abec380b46 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -13,6 +13,8 @@ import { Helpers } from "@budibase/bbui" import { RelationshipErrorChecker } from "./relationshipErrors" import { onMount } from "svelte" + import RelationshipSelector from "components/common/RelationshipSelector.svelte" + import { PrettyRelationshipDefinitions } from "constants/backend" export let save export let datasource @@ -22,16 +24,21 @@ export let selectedFromTable export let close - const relationshipTypes = [ - { - label: "One to Many", - value: RelationshipType.MANY_TO_ONE, + let relationshipMap = { + [RelationshipType.MANY_TO_MANY]: { + part1: PrettyRelationshipDefinitions.MANY, + part2: PrettyRelationshipDefinitions.MANY, }, - { - label: "Many to Many", - value: RelationshipType.MANY_TO_MANY, + [RelationshipType.MANY_TO_ONE]: { + part1: PrettyRelationshipDefinitions.ONE, + part2: PrettyRelationshipDefinitions.MANY, }, - ] + } + let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions) + let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions) + + let relationshipPart1 = PrettyRelationshipDefinitions.MANY + let relationshipPart2 = PrettyRelationshipDefinitions.ONE let originalFromColumnName = toRelationship.name, originalToColumnName = fromRelationship.name @@ -49,14 +56,32 @@ ) let errors = {} let fromPrimary, fromForeign, fromColumn, toColumn - let fromId, toId, throughId, throughToKey, throughFromKey + + let throughId, throughToKey, throughFromKey let isManyToMany, isManyToOne, relationshipType let hasValidated = false + $: fromId = null + $: toId = null + $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id, + name: table.name, + _id: table._id, })) + + $: { + // Determine the relationship type based on the selected values of both parts + relationshipType = Object.entries(relationshipMap).find( + ([_, parts]) => + parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2 + )?.[0] + + changed(() => { + hasValidated = false + }) + } $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType) $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY @@ -338,33 +363,34 @@ onConfirm={saveRelationship} disabled={!valid} > - - changed(() => { - const table = plusTables.find(tbl => tbl._id === e.detail) - fromColumn = table?.name || "" - fromPrimary = table?.primary?.[0] - })} - /> - {/if} + + + changed(() => { + const table = plusTables.find(tbl => tbl._id === e.detail) + fromColumn = table?.name || "" + fromPrimary = table?.primary?.[0] + })} + secondaryTableChanged={e => + changed(() => { + const table = plusTables.find(tbl => tbl._id === e.detail) + toColumn = table.name || "" + fromForeign = null + })} + /> + {#if isManyToOne && fromId} - changed(() => { - const table = plusTables.find(tbl => tbl._id === e.detail) - toColumn = table.name || "" - fromForeign = null - })} - /> {#if isManyToMany} table.name} getOptionValue={table => table._id} bind:value={relationshipTableIdPrimary} + on:change={primaryTableChanged} + bind:error={errors.fromTable} /> @@ -46,20 +52,24 @@ +{#if editableColumn} + +{/if}