diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 6e886f3011..e940e6fa10 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -7,7 +7,6 @@ on: branches: - master - develop - - new-design-ui pull_request: branches: - master @@ -60,19 +59,3 @@ jobs: with: install: false command: yarn test:e2e:ci - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Upload to S3 - if: github.ref == 'refs/heads/new-design-ui' - run: | - tar -czvf new_ui.tar.gz packages/server/builder/assets packages/server/builder/index.html - aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/ - aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js - aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*" - diff --git a/.github/workflows/deploy-single-image.yml b/.github/workflows/deploy-single-image.yml new file mode 100644 index 0000000000..0bd5c71a40 --- /dev/null +++ b/.github/workflows/deploy-single-image.yml @@ -0,0 +1,59 @@ +name: Deploy Budibase Single Container Image to DockerHub + +on: + workflow_dispatch: + +env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref}} + BRANCH: ${{ github.event.pull_request.head.ref }} + CI: true + PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com +jobs: + build: + name: "build" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [14.x] + steps: + - name: "Checkout" + uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v1 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Install Pro + run: yarn install:pro $BRANCH $BASE_BRANCH + - name: Run Yarn + run: yarn + - name: Run Yarn Bootstrap + run: yarn bootstrap + - name: Runt Yarn Lint + run: yarn lint + - name: Run Yarn Build + run: yarn build:docker:pre + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Budibase service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }} + file: ./hosting/single/Dockerfile diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f33dcc6d53..348b600f90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,6 +16,16 @@ on: - 'package.json' - 'yarn.lock' workflow_dispatch: + inputs: + versioning: + type: choice + description: "Versioning type: patch, minor, major" + default: patch + options: + - patch + - minor + - major + required: true env: # Posthog token used by ui at build time @@ -58,6 +68,7 @@ jobs: - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + RELEASE_VERSION_TYPE: ${{ github.event.inputs.versioning }} run: | # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default git config --global user.name "Budibase Release Bot" diff --git a/.yarnrc b/.yarnrc new file mode 100644 index 0000000000..21fa517e23 --- /dev/null +++ b/.yarnrc @@ -0,0 +1 @@ +network-timeout 100000 diff --git a/README.md b/README.md index e8c6475d90..ae149f7347 100644 --- a/README.md +++ b/README.md @@ -135,13 +135,18 @@ You can learn more about the Budibase API at the following places: ## 🏁 Get started - - Deploy Budibase self-hosted in your existing infrastructure, using Docker, Kubernetes, and Digital Ocean. Or use Budibase Cloud if you don't need to self-host, and would like to get started quickly. ### [Get started with self-hosting Budibase](https://docs.budibase.com/docs/hosting-methods) +- [Docker - single ARM compatible image](https://docs.budibase.com/docs/docker) +- [Docker Compose](https://docs.budibase.com/docs/docker-compose) +- [Kubernetes](https://docs.budibase.com/docs/kubernetes-k8s) +- [Digital Ocean](https://docs.budibase.com/docs/digitalocean) +- [Portainer](https://docs.budibase.com/docs/portainer) + + ### [Get started with Budibase Cloud](https://budibase.com) diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index ddc725d302..7a2c483cc8 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -122,6 +122,14 @@ spec: value: {{ .Values.globals.automationMaxIterations | quote }} - name: TENANT_FEATURE_FLAGS value: {{ .Values.globals.tenantFeatureFlags | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: { { .Values.globals.bbAdminUserEmail | quote } } + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: { { .Values.globals.bbAdminUserPassword | quote } } + {{ end }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/hosting/.env b/hosting/.env index 39df76d01e..11dd661bf1 100644 --- a/hosting/.env +++ b/hosting/.env @@ -18,4 +18,8 @@ MINIO_PORT=4004 COUCH_DB_PORT=4005 REDIS_PORT=6379 WATCHTOWER_PORT=6161 -BUDIBASE_ENVIRONMENT=PRODUCTION \ No newline at end of file +BUDIBASE_ENVIRONMENT=PRODUCTION + +# An admin user can be automatically created initially if these are set +BB_ADMIN_USER_EMAIL= +BB_ADMIN_USER_PASSWORD= \ No newline at end of file diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index cdbe2cb66c..f669f9261d 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -23,6 +23,8 @@ services: ENABLE_ANALYTICS: "true" REDIS_URL: redis-service:6379 REDIS_PASSWORD: ${REDIS_PASSWORD} + BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} + BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} depends_on: - worker-service - redis-service diff --git a/hosting/hosting.properties b/hosting/hosting.properties index c8e2f5c606..11dd661bf1 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -19,3 +19,7 @@ COUCH_DB_PORT=4005 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION + +# An admin user can be automatically created initially if these are set +BB_ADMIN_USER_EMAIL= +BB_ADMIN_USER_PASSWORD= \ No newline at end of file diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 5e1b0b1374..772ae2a8ab 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -34,27 +34,32 @@ ENV \ ARCHITECTURE=amd \ BUDIBASE_ENVIRONMENT=PRODUCTION \ CLUSTER_PORT=80 \ - COUCHDB_PASSWORD=budibase \ - COUCHDB_USER=budibase \ - COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ # CUSTOM_DOMAIN=budi001.custom.com \ DEPLOYMENT_ENVIRONMENT=docker \ - INTERNAL_API_KEY=budibase \ - JWT_SECRET=testsecret \ - MINIO_ACCESS_KEY=budibase \ - MINIO_SECRET_KEY=budibase \ MINIO_URL=http://localhost:9000 \ POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \ - REDIS_PASSWORD=budibase \ REDIS_URL=localhost:6379 \ SELF_HOSTED=1 \ TARGETBUILD=$TARGETBUILD \ WORKER_PORT=4002 \ - WORKER_URL=http://localhost:4002 + WORKER_URL=http://localhost:4002 \ + APPS_URL=http://localhost:4001 + +# These secret env variables are generated by the runner at startup +# their values can be overriden by the user, they will be written +# to the .env file in the /data directory for use later on +# REDIS_PASSWORD=budibase \ +# COUCHDB_PASSWORD=budibase \ +# COUCHDB_USER=budibase \ +# COUCH_DB_URL=http://budibase:budibase@localhost:5984 \ +# INTERNAL_API_KEY=budibase \ +# JWT_SECRET=testsecret \ +# MINIO_ACCESS_KEY=budibase \ +# MINIO_SECRET_KEY=budibase \ # install base dependencies RUN apt-get update && \ - apt-get install -y software-properties-common wget nginx && \ + apt-get install -y software-properties-common wget nginx uuid-runtime && \ apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ apt-get update @@ -66,8 +71,8 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh & npm install --global yarn pm2 # setup nginx -ADD hosting/single/nginx.conf /etc/nginx -ADD hosting/single/nginx-default-site.conf /etc/nginx/sites-enabled/default +ADD hosting/single/nginx/nginx.conf /etc/nginx +ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/nginx && \ touch /var/log/nginx/error.log && \ touch /var/run/nginx.pid @@ -86,13 +91,13 @@ RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clou WORKDIR /opt/clouseau RUN mkdir ./bin -ADD hosting/single/clouseau ./bin/ -ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./ +ADD hosting/single/clouseau/clouseau ./bin/ +ADD hosting/single/clouseau/log4j.properties hosting/single/clouseau/clouseau.ini ./ RUN chmod +x ./bin/clouseau # setup CouchDB WORKDIR /opt/couchdb -ADD hosting/single/vm.args ./etc/ +ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/ # setup minio WORKDIR /minio diff --git a/hosting/single/clouseau b/hosting/single/clouseau/clouseau similarity index 100% rename from hosting/single/clouseau rename to hosting/single/clouseau/clouseau diff --git a/hosting/single/clouseau.ini b/hosting/single/clouseau/clouseau.ini similarity index 92% rename from hosting/single/clouseau.ini rename to hosting/single/clouseau/clouseau.ini index f086cf0398..78e43744e5 100644 --- a/hosting/single/clouseau.ini +++ b/hosting/single/clouseau/clouseau.ini @@ -7,7 +7,7 @@ name=clouseau@127.0.0.1 cookie=monster ; the path where you would like to store the search index files -dir=/opt/couchdb/data/search +dir=/data/search ; the number of search indexes that can be open simultaneously max_indexes_open=500 diff --git a/hosting/single/log4j.properties b/hosting/single/clouseau/log4j.properties similarity index 100% rename from hosting/single/log4j.properties rename to hosting/single/clouseau/log4j.properties diff --git a/hosting/single/couch/local.ini b/hosting/single/couch/local.ini new file mode 100644 index 0000000000..72872a60e1 --- /dev/null +++ b/hosting/single/couch/local.ini @@ -0,0 +1,5 @@ +; CouchDB Configuration Settings + +[couchdb] +database_dir = /data/couch/dbs +view_index_dir = /data/couch/views diff --git a/hosting/single/vm.args b/hosting/single/couch/vm.args similarity index 100% rename from hosting/single/vm.args rename to hosting/single/couch/vm.args diff --git a/hosting/single/nginx-default-site.conf b/hosting/single/nginx/nginx-default-site.conf similarity index 99% rename from hosting/single/nginx-default-site.conf rename to hosting/single/nginx/nginx-default-site.conf index 964313fa73..c0d80a0185 100644 --- a/hosting/single/nginx-default-site.conf +++ b/hosting/single/nginx/nginx-default-site.conf @@ -88,7 +88,4 @@ server { gzip_proxied any; gzip_comp_level 6; gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; - - - } diff --git a/hosting/single/nginx.conf b/hosting/single/nginx/nginx.conf similarity index 100% rename from hosting/single/nginx.conf rename to hosting/single/nginx/nginx.conf diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 6f3d247842..f8c1fc5e56 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,6 +1,34 @@ +#!/bin/bash +declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD") +if [ -f "/data/.env" ]; then + export $(cat /data/.env | xargs) +fi +# first randomise any unset environment variables +for ENV_VAR in "${ENV_VARS[@]}" +do + temp=$(eval "echo \$$ENV_VAR") + if [[ -z "${temp}" ]]; then + eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" + fi +done +if [[ -z "${COUCH_DB_URL}" ]]; then + export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 +fi +if [ ! -f "/data/.env" ]; then + touch /data/.env + for ENV_VAR in "${ENV_VARS[@]}" + do + temp=$(eval "echo \$$ENV_VAR") + echo "$ENV_VAR=$temp" >> /data/.env + done +fi + +# make these directories in runner, incase of mount +mkdir -p /data/couch/dbs /data/couch/views +chown couchdb:couchdb /data/couch /data/couch/dbs /data/couch/views redis-server --requirepass $REDIS_PASSWORD & /opt/clouseau/bin/clouseau & -/minio/minio server /minio & +/minio/minio server /data/minio & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then diff --git a/hosting/single/test.sh b/hosting/single/test.sh index c7ef53f994..8830426a47 100755 --- a/hosting/single/test.sh +++ b/hosting/single/test.sh @@ -1,4 +1,4 @@ #!/bin/bash -id=$(docker run -t -d -p 80:80 budibase:latest) +id=$(docker run -t -d -p 8080:80 budibase:latest) docker exec -it $id bash docker kill $id diff --git a/lerna.json b/lerna.json index 89ef3fcccf..67cbc31eff 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.219-alpha.10", + "version": "1.1.10-alpha.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 9c35af497f..0c7d3989a2 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh", "build": "lerna run build", "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput", - "release": "lerna publish patch --yes --force-publish && yarn release:pro", + "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop", "release:pro": "bash scripts/pro/release.sh", "release:pro:develop": "bash scripts/pro/release.sh develop", @@ -40,7 +40,8 @@ "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1", "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", - "test": "lerna run test", + "test": "lerna run test && yarn test:pro", + "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", @@ -53,6 +54,7 @@ "test:e2e:ci:notify": "lerna run cy:ci:notify", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker:pre": "lerna run build && lerna run predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy", "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy", @@ -64,7 +66,7 @@ "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "lerna run build && lerna run predocker && npm run build:docker:single:image", + "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", diff --git a/packages/backend-core/cache.js b/packages/backend-core/cache.js index 6b319357c4..c8bd3c9b6f 100644 --- a/packages/backend-core/cache.js +++ b/packages/backend-core/cache.js @@ -5,4 +5,5 @@ module.exports = { app: require("./src/cache/appMetadata"), writethrough: require("./src/cache/writethrough"), ...generic, + cache: generic, } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 01f0b319c1..1131086a2f 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.219-alpha.10", + "version": "1.1.10-alpha.4", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^1.0.219-alpha.10", + "@budibase/types": "^1.1.10-alpha.4", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", @@ -62,6 +62,7 @@ "@shopify/jest-koa-mocks": "3.1.5", "@types/jest": "27.5.1", "@types/koa": "2.0.52", + "@types/lodash": "4.14.180", "@types/node": "14.18.20", "@types/node-fetch": "2.6.1", "@types/pouchdb": "6.4.0", diff --git a/packages/backend-core/src/context/constants.ts b/packages/backend-core/src/context/constants.ts new file mode 100644 index 0000000000..ef8dcd7821 --- /dev/null +++ b/packages/backend-core/src/context/constants.ts @@ -0,0 +1,17 @@ +export enum ContextKeys { + TENANT_ID = "tenantId", + GLOBAL_DB = "globalDb", + APP_ID = "appId", + IDENTITY = "identity", + // whatever the request app DB was + CURRENT_DB = "currentDb", + // get the prod app DB from the request + PROD_DB = "prodDb", + // get the dev app DB from the request + DEV_DB = "devDb", + DB_OPTS = "dbOpts", + // check if something else is using the context, don't close DB + TENANCY_IN_USE = "tenancyInUse", + APP_IN_USE = "appInUse", + IDENTITY_IN_USE = "identityInUse", +} diff --git a/packages/backend-core/src/context/index.js b/packages/backend-core/src/context/index.js deleted file mode 100644 index bd4d857ef2..0000000000 --- a/packages/backend-core/src/context/index.js +++ /dev/null @@ -1,354 +0,0 @@ -const env = require("../environment") -const { SEPARATOR, DocumentTypes } = require("../db/constants") -const { DEFAULT_TENANT_ID } = require("../constants") -const cls = require("./FunctionContext") -const { dangerousGetDB, closeDB } = require("../db") -const { getProdAppID, getDevelopmentAppID } = require("../db/conversions") -const { baseGlobalDBName } = require("../tenancy/utils") -const { isEqual } = require("lodash") - -// some test cases call functions directly, need to -// store an app ID to pretend there is a context -let TEST_APP_ID = null - -const ContextKeys = { - TENANT_ID: "tenantId", - GLOBAL_DB: "globalDb", - APP_ID: "appId", - IDENTITY: "identity", - // whatever the request app DB was - CURRENT_DB: "currentDb", - // get the prod app DB from the request - PROD_DB: "prodDb", - // get the dev app DB from the request - DEV_DB: "devDb", - DB_OPTS: "dbOpts", - // check if something else is using the context, don't close DB - IN_USE: "inUse", -} - -exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID - -// this function makes sure the PouchDB objects are closed and -// fully deleted when finished - this protects against memory leaks -async function closeAppDBs() { - const dbKeys = [ - ContextKeys.CURRENT_DB, - ContextKeys.PROD_DB, - ContextKeys.DEV_DB, - ] - for (let dbKey of dbKeys) { - const db = cls.getFromContext(dbKey) - if (!db) { - continue - } - await closeDB(db) - // clear the DB from context, incase someone tries to use it again - cls.setOnContext(dbKey, null) - } - // clear the app ID now that the databases are closed - if (cls.getFromContext(ContextKeys.APP_ID)) { - cls.setOnContext(ContextKeys.APP_ID, null) - } - if (cls.getFromContext(ContextKeys.DB_OPTS)) { - cls.setOnContext(ContextKeys.DB_OPTS, null) - } -} - -exports.closeTenancy = async () => { - if (env.USE_COUCH) { - await closeDB(exports.getGlobalDB()) - } - // clear from context now that database is closed/task is finished - cls.setOnContext(ContextKeys.TENANT_ID, null) - cls.setOnContext(ContextKeys.GLOBAL_DB, null) -} - -exports.isDefaultTenant = () => { - return exports.getTenantId() === exports.DEFAULT_TENANT_ID -} - -exports.isMultiTenant = () => { - return env.MULTI_TENANCY -} - -// used for automations, API endpoints should always be in context already -exports.doInTenant = (tenantId, task, { forceNew } = {}) => { - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the tenant id - if (!opts.existing) { - exports.updateTenantId(tenantId) - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await exports.closeTenancy() - } else { - cls.setOnContext(using - 1) - } - } - } - - const using = cls.getFromContext(ContextKeys.IN_USE) - if ( - !forceNew && - using && - cls.getFromContext(ContextKeys.TENANT_ID) === tenantId - ) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -/** - * Given an app ID this will attempt to retrieve the tenant ID from it. - * @return {null|string} The tenant ID found within the app ID. - */ -exports.getTenantIDFromAppID = appId => { - if (!appId) { - return null - } - const split = appId.split(SEPARATOR) - const hasDev = split[1] === DocumentTypes.DEV - if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { - return null - } - if (hasDev) { - return split[2] - } else { - return split[1] - } -} - -const setAppTenantId = appId => { - const appTenantId = - exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID - exports.updateTenantId(appTenantId) -} - -exports.doInAppContext = (appId, task, { forceNew } = {}) => { - if (!appId) { - throw new Error("appId is required") - } - - const identity = exports.getIdentity() - - // the internal function is so that we can re-use an existing - // context - don't want to close DB on a parent context - async function internal(opts = { existing: false }) { - // set the app tenant id - if (!opts.existing) { - setAppTenantId(appId) - } - // set the app ID - cls.setOnContext(ContextKeys.APP_ID, appId) - // preserve the identity - exports.setIdentity(identity) - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - await closeAppDBs() - } else { - cls.setOnContext(using - 1) - } - } - } - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal() - }) - } -} - -exports.doInIdentityContext = (identity, task) => { - if (!identity) { - throw new Error("identity is required") - } - - async function internal(opts = { existing: false }) { - if (!opts.existing) { - cls.setOnContext(ContextKeys.IDENTITY, identity) - // set the tenant so that doInTenant will preserve identity - if (identity.tenantId) { - exports.updateTenantId(identity.tenantId) - } - } - - try { - // invoke the task - return await task() - } finally { - const using = cls.getFromContext(ContextKeys.IN_USE) - if (!using || using <= 1) { - exports.setIdentity(null) - } else { - cls.setOnContext(using - 1) - } - } - } - - const existing = cls.getFromContext(ContextKeys.IDENTITY) - const using = cls.getFromContext(ContextKeys.IN_USE) - if (using && existing && existing._id === identity._id) { - cls.setOnContext(ContextKeys.IN_USE, using + 1) - return internal({ existing: true }) - } else { - return cls.run(async () => { - cls.setOnContext(ContextKeys.IN_USE, 1) - return internal({ existing: false }) - }) - } -} - -exports.setIdentity = identity => { - cls.setOnContext(ContextKeys.IDENTITY, identity) -} - -exports.getIdentity = () => { - try { - return cls.getFromContext(ContextKeys.IDENTITY) - } catch (e) { - // do nothing - identity is not in context - } -} - -exports.updateTenantId = tenantId => { - cls.setOnContext(ContextKeys.TENANT_ID, tenantId) - if (env.USE_COUCH) { - exports.setGlobalDB(tenantId) - } -} - -exports.updateAppId = async appId => { - try { - // have to close first, before removing the databases from context - await closeAppDBs() - cls.setOnContext(ContextKeys.APP_ID, appId) - } catch (err) { - if (env.isTest()) { - TEST_APP_ID = appId - } else { - throw err - } - } -} - -exports.setGlobalDB = tenantId => { - const dbName = baseGlobalDBName(tenantId) - const db = dangerousGetDB(dbName) - cls.setOnContext(ContextKeys.GLOBAL_DB, db) - return db -} - -exports.getGlobalDB = () => { - const db = cls.getFromContext(ContextKeys.GLOBAL_DB) - if (!db) { - throw new Error("Global DB not found") - } - return db -} - -exports.isTenantIdSet = () => { - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - return !!tenantId -} - -exports.getTenantId = () => { - if (!exports.isMultiTenant()) { - return exports.DEFAULT_TENANT_ID - } - const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) - if (!tenantId) { - throw new Error("Tenant id not found") - } - return tenantId -} - -exports.getAppId = () => { - const foundId = cls.getFromContext(ContextKeys.APP_ID) - if (!foundId && env.isTest() && TEST_APP_ID) { - return TEST_APP_ID - } else { - return foundId - } -} - -function getContextDB(key, opts) { - const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` - let storedOpts = cls.getFromContext(dbOptsKey) - let db = cls.getFromContext(key) - if (db && isEqual(opts, storedOpts)) { - return db - } - - const appId = exports.getAppId() - let toUseAppId - - switch (key) { - case ContextKeys.CURRENT_DB: - toUseAppId = appId - break - case ContextKeys.PROD_DB: - toUseAppId = getProdAppID(appId) - break - case ContextKeys.DEV_DB: - toUseAppId = getDevelopmentAppID(appId) - break - } - - db = dangerousGetDB(toUseAppId, opts) - try { - cls.setOnContext(key, db) - if (opts) { - cls.setOnContext(dbOptsKey, opts) - } - } catch (err) { - if (!env.isTest()) { - throw err - } - } - return db -} - -/** - * Opens the app database based on whatever the request - * contained, dev or prod. - */ -exports.getAppDB = (opts = null) => { - return getContextDB(ContextKeys.CURRENT_DB, opts) -} - -/** - * This specifically gets the prod app ID, if the request - * contained a development app ID, this will open the prod one. - */ -exports.getProdAppDB = (opts = null) => { - return getContextDB(ContextKeys.PROD_DB, opts) -} - -/** - * This specifically gets the dev app ID, if the request - * contained a prod app ID, this will open the dev one. - */ -exports.getDevAppDB = (opts = null) => { - return getContextDB(ContextKeys.DEV_DB, opts) -} diff --git a/packages/backend-core/src/context/index.ts b/packages/backend-core/src/context/index.ts new file mode 100644 index 0000000000..e0db18dde6 --- /dev/null +++ b/packages/backend-core/src/context/index.ts @@ -0,0 +1,247 @@ +import env from "../environment" +import { SEPARATOR, DocumentTypes } from "../db/constants" +import cls from "./FunctionContext" +import { dangerousGetDB, closeDB } from "../db" +import { baseGlobalDBName } from "../tenancy/utils" +import { IdentityContext } from "@budibase/types" +import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants" +import { ContextKeys } from "./constants" +import { + updateUsing, + closeWithUsing, + setAppTenantId, + setIdentity, + closeAppDBs, + getContextDB, +} from "./utils" + +export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID + +// some test cases call functions directly, need to +// store an app ID to pretend there is a context +let TEST_APP_ID: string | null = null + +export const closeTenancy = async () => { + let db + try { + if (env.USE_COUCH) { + db = getGlobalDB() + } + } catch (err) { + // no DB found - skip closing + return + } + await closeDB(db) + // clear from context now that database is closed/task is finished + cls.setOnContext(ContextKeys.TENANT_ID, null) + cls.setOnContext(ContextKeys.GLOBAL_DB, null) +} + +// export const isDefaultTenant = () => { +// return getTenantId() === DEFAULT_TENANT_ID +// } + +export const isMultiTenant = () => { + return env.MULTI_TENANCY +} + +/** + * Given an app ID this will attempt to retrieve the tenant ID from it. + * @return {null|string} The tenant ID found within the app ID. + */ +export const getTenantIDFromAppID = (appId: string) => { + if (!appId) { + return null + } + const split = appId.split(SEPARATOR) + const hasDev = split[1] === DocumentTypes.DEV + if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) { + return null + } + if (hasDev) { + return split[2] + } else { + return split[1] + } +} + +// used for automations, API endpoints should always be in context already +export const doInTenant = (tenantId: string | null, task: any) => { + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the tenant id + global db if this is a new context + if (!opts.existing) { + updateTenantId(tenantId) + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.TENANCY_IN_USE, () => { + return closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKeys.TENANT_ID) === tenantId + return updateUsing(ContextKeys.TENANCY_IN_USE, existing, internal) +} + +export const doInAppContext = (appId: string, task: any) => { + if (!appId) { + throw new Error("appId is required") + } + + const identity = getIdentity() + + // the internal function is so that we can re-use an existing + // context - don't want to close DB on a parent context + async function internal(opts = { existing: false }) { + // set the app tenant id + if (!opts.existing) { + setAppTenantId(appId) + } + // set the app ID + cls.setOnContext(ContextKeys.APP_ID, appId) + + // preserve the identity + if (identity) { + setIdentity(identity) + } + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.APP_IN_USE, async () => { + await closeAppDBs() + await closeTenancy() + }) + } + } + const existing = cls.getFromContext(ContextKeys.APP_ID) === appId + return updateUsing(ContextKeys.APP_IN_USE, existing, internal) +} + +export const doInIdentityContext = (identity: IdentityContext, task: any) => { + if (!identity) { + throw new Error("identity is required") + } + + async function internal(opts = { existing: false }) { + if (!opts.existing) { + cls.setOnContext(ContextKeys.IDENTITY, identity) + // set the tenant so that doInTenant will preserve identity + if (identity.tenantId) { + updateTenantId(identity.tenantId) + } + } + + try { + // invoke the task + return await task() + } finally { + await closeWithUsing(ContextKeys.IDENTITY_IN_USE, async () => { + setIdentity(null) + await closeTenancy() + }) + } + } + + const existing = cls.getFromContext(ContextKeys.IDENTITY) + return updateUsing(ContextKeys.IDENTITY_IN_USE, existing, internal) +} + +export const getIdentity = (): IdentityContext | undefined => { + try { + return cls.getFromContext(ContextKeys.IDENTITY) + } catch (e) { + // do nothing - identity is not in context + } +} + +export const updateTenantId = (tenantId: string | null) => { + cls.setOnContext(ContextKeys.TENANT_ID, tenantId) + if (env.USE_COUCH) { + setGlobalDB(tenantId) + } +} + +export const updateAppId = async (appId: string) => { + try { + // have to close first, before removing the databases from context + await closeAppDBs() + cls.setOnContext(ContextKeys.APP_ID, appId) + } catch (err) { + if (env.isTest()) { + TEST_APP_ID = appId + } else { + throw err + } + } +} + +export const setGlobalDB = (tenantId: string | null) => { + const dbName = baseGlobalDBName(tenantId) + const db = dangerousGetDB(dbName) + cls.setOnContext(ContextKeys.GLOBAL_DB, db) + return db +} + +export const getGlobalDB = () => { + const db = cls.getFromContext(ContextKeys.GLOBAL_DB) + if (!db) { + throw new Error("Global DB not found") + } + return db +} + +export const isTenantIdSet = () => { + const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) + return !!tenantId +} + +export const getTenantId = () => { + if (!isMultiTenant()) { + return DEFAULT_TENANT_ID + } + const tenantId = cls.getFromContext(ContextKeys.TENANT_ID) + if (!tenantId) { + throw new Error("Tenant id not found") + } + return tenantId +} + +export const getAppId = () => { + const foundId = cls.getFromContext(ContextKeys.APP_ID) + if (!foundId && env.isTest() && TEST_APP_ID) { + return TEST_APP_ID + } else { + return foundId + } +} + +/** + * Opens the app database based on whatever the request + * contained, dev or prod. + */ +export const getAppDB = (opts?: any) => { + return getContextDB(ContextKeys.CURRENT_DB, opts) +} + +/** + * This specifically gets the prod app ID, if the request + * contained a development app ID, this will open the prod one. + */ +export const getProdAppDB = (opts?: any) => { + return getContextDB(ContextKeys.PROD_DB, opts) +} + +/** + * This specifically gets the dev app ID, if the request + * contained a prod app ID, this will open the dev one. + */ +export const getDevAppDB = (opts?: any) => { + return getContextDB(ContextKeys.DEV_DB, opts) +} diff --git a/packages/backend-core/src/context/tests/index.spec.ts b/packages/backend-core/src/context/tests/index.spec.ts new file mode 100644 index 0000000000..55ecd333a3 --- /dev/null +++ b/packages/backend-core/src/context/tests/index.spec.ts @@ -0,0 +1,148 @@ +import "../../../tests/utilities/TestConfiguration" +import * as context from ".." +import { DEFAULT_TENANT_ID } from "../../constants" +import env from "../../environment" + +// must use require to spy index file exports due to known issue in jest +const dbUtils = require("../../db") +jest.spyOn(dbUtils, "closeDB") +jest.spyOn(dbUtils, "dangerousGetDB") + +describe("context", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("doInTenant", () => { + describe("single-tenancy", () => { + it("defaults to the default tenant", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe(DEFAULT_TENANT_ID) + }) + + it("defaults to the default tenant db", async () => { + await context.doInTenant(DEFAULT_TENANT_ID, () => { + const db = context.getGlobalDB() + expect(db.name).toBe("global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + }) + + describe("multi-tenancy", () => { + beforeEach(() => { + env._set("MULTI_TENANCY", 1) + }) + + it("fails when no tenant id is set", () => { + const test = () => { + let error + try { + context.getTenantId() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Tenant id not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("fails when no tenant db is set", () => { + const test = () => { + let error + try { + context.getGlobalDB() + } catch (e: any) { + error = e + } + expect(error.message).toBe("Global DB not found") + } + + // test under no tenancy + test() + + // test after tenancy has been accessed to ensure cleanup + context.doInTenant("test", () => {}) + test() + }) + + it("sets tenant id", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + + it("initialises the tenant db", async () => { + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets the tenant id when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", async () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + await context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + }) + }) + }) + }) + + it("initialises the tenant db when nested with same tenant id", async () => { + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", async () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + + await context.doInTenant("test", () => { + const db = context.getGlobalDB() + expect(db.name).toBe("test_global-db") + }) + }) + }) + + // only 1 db is opened and closed + expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1) + expect(dbUtils.closeDB).toHaveBeenCalledTimes(1) + }) + + it("sets different tenant id inside another context", () => { + context.doInTenant("test", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("test") + + context.doInTenant("nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("nested") + + context.doInTenant("double-nested", () => { + const tenantId = context.getTenantId() + expect(tenantId).toBe("double-nested") + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/backend-core/src/context/utils.ts b/packages/backend-core/src/context/utils.ts new file mode 100644 index 0000000000..62693f18e8 --- /dev/null +++ b/packages/backend-core/src/context/utils.ts @@ -0,0 +1,113 @@ +import { + DEFAULT_TENANT_ID, + getAppId, + getTenantIDFromAppID, + updateTenantId, +} from "./index" +import cls from "./FunctionContext" +import { IdentityContext } from "@budibase/types" +import { ContextKeys } from "./constants" +import { dangerousGetDB, closeDB } from "../db" +import { isEqual } from "lodash" +import { getDevelopmentAppID, getProdAppID } from "../db/conversions" +import env from "../environment" + +export async function updateUsing( + usingKey: string, + existing: boolean, + internal: (opts: { existing: boolean }) => Promise +) { + const using = cls.getFromContext(usingKey) + if (using && existing) { + cls.setOnContext(usingKey, using + 1) + return internal({ existing: true }) + } else { + return cls.run(async () => { + cls.setOnContext(usingKey, 1) + return internal({ existing: false }) + }) + } +} + +export async function closeWithUsing( + usingKey: string, + closeFn: () => Promise +) { + const using = cls.getFromContext(usingKey) + if (!using || using <= 1) { + await closeFn() + } else { + cls.setOnContext(usingKey, using - 1) + } +} + +export const setAppTenantId = (appId: string) => { + const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID + updateTenantId(appTenantId) +} + +export const setIdentity = (identity: IdentityContext | null) => { + cls.setOnContext(ContextKeys.IDENTITY, identity) +} + +// this function makes sure the PouchDB objects are closed and +// fully deleted when finished - this protects against memory leaks +export async function closeAppDBs() { + const dbKeys = [ + ContextKeys.CURRENT_DB, + ContextKeys.PROD_DB, + ContextKeys.DEV_DB, + ] + for (let dbKey of dbKeys) { + const db = cls.getFromContext(dbKey) + if (!db) { + continue + } + await closeDB(db) + // clear the DB from context, incase someone tries to use it again + cls.setOnContext(dbKey, null) + } + // clear the app ID now that the databases are closed + if (cls.getFromContext(ContextKeys.APP_ID)) { + cls.setOnContext(ContextKeys.APP_ID, null) + } + if (cls.getFromContext(ContextKeys.DB_OPTS)) { + cls.setOnContext(ContextKeys.DB_OPTS, null) + } +} + +export function getContextDB(key: string, opts: any) { + const dbOptsKey = `${key}${ContextKeys.DB_OPTS}` + let storedOpts = cls.getFromContext(dbOptsKey) + let db = cls.getFromContext(key) + if (db && isEqual(opts, storedOpts)) { + return db + } + + const appId = getAppId() + let toUseAppId + + switch (key) { + case ContextKeys.CURRENT_DB: + toUseAppId = appId + break + case ContextKeys.PROD_DB: + toUseAppId = getProdAppID(appId) + break + case ContextKeys.DEV_DB: + toUseAppId = getDevelopmentAppID(appId) + break + } + db = dangerousGetDB(toUseAppId, opts) + try { + cls.setOnContext(key, db) + if (opts) { + cls.setOnContext(dbOptsKey, opts) + } + } catch (err) { + if (!env.isTest()) { + throw err + } + } + return db +} diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index be0e824e61..716762dd45 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -11,7 +11,7 @@ export enum AutomationViewModes { } export enum ViewNames { - USER_BY_EMAIL = "by_email", + USER_BY_EMAIL = "by_email2", BY_API_KEY = "by_api_key", USER_BY_BUILDERS = "by_builders", LINK = "by_link", @@ -19,6 +19,13 @@ export enum ViewNames { AUTOMATION_LOGS = "automation_logs", } +export const DeprecatedViews = { + [ViewNames.USER_BY_EMAIL]: [ + // removed due to inaccuracy in view doc filter logic + "by_email", + ], +} + export enum DocumentTypes { USER = "us", WORKSPACE = "workspace", diff --git a/packages/backend-core/src/db/index.js b/packages/backend-core/src/db/index.js index 8124be979e..aa6f7ebc2c 100644 --- a/packages/backend-core/src/db/index.js +++ b/packages/backend-core/src/db/index.js @@ -1,10 +1,18 @@ const pouch = require("./pouch") const env = require("../environment") +const openDbs = [] let PouchDB let initialised = false const dbList = new Set() +if (env.MEMORY_LEAK_CHECK) { + setInterval(() => { + console.log("--- OPEN DBS ---") + console.log(openDbs) + }, 5000) +} + const put = dbPut => async (doc, options = {}) => { @@ -35,6 +43,9 @@ exports.dangerousGetDB = (dbName, opts) => { dbList.add(dbName) } const db = new PouchDB(dbName, opts) + if (env.MEMORY_LEAK_CHECK) { + openDbs.push(db.name) + } const dbPut = db.put db.put = put(dbPut) return db @@ -46,6 +57,9 @@ exports.closeDB = async db => { if (!db || env.isTest()) { return } + if (env.MEMORY_LEAK_CHECK) { + openDbs.splice(openDbs.indexOf(db.name), 1) + } try { // specifically await so that if there is an error, it can be ignored return await db.close() diff --git a/packages/backend-core/src/db/views.js b/packages/backend-core/src/db/views.js index e0281c6584..1e8dd7ee77 100644 --- a/packages/backend-core/src/db/views.js +++ b/packages/backend-core/src/db/views.js @@ -1,20 +1,42 @@ -const { DocumentTypes, ViewNames } = require("./utils") +const { + DocumentTypes, + ViewNames, + DeprecatedViews, + SEPARATOR, +} = require("./utils") const { getGlobalDB } = require("../tenancy") +const DESIGN_DB = "_design/database" + function DesignDoc() { return { - _id: "_design/database", + _id: DESIGN_DB, // view collation information, read before writing any complex views: // https://docs.couchdb.org/en/master/ddocs/views/collation.html#collation-specification views: {}, } } -exports.createUserEmailView = async () => { +async function removeDeprecated(db, viewName) { + if (!DeprecatedViews[viewName]) { + return + } + try { + const designDoc = await db.get(DESIGN_DB) + for (let deprecatedNames of DeprecatedViews[viewName]) { + delete designDoc.views[deprecatedNames] + } + await db.put(designDoc) + } catch (err) { + // doesn't exist, ignore + } +} + +exports.createNewUserEmailView = async () => { const db = getGlobalDB() let designDoc try { - designDoc = await db.get("_design/database") + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() @@ -22,7 +44,7 @@ exports.createUserEmailView = async () => { const view = { // if using variables in a map function need to inject them before use map: `function(doc) { - if (doc._id.startsWith("${DocumentTypes.USER}")) { + if (doc._id.startsWith("${DocumentTypes.USER}${SEPARATOR}")) { emit(doc.email.toLowerCase(), doc._id) } }`, @@ -81,7 +103,7 @@ exports.createUserBuildersView = async () => { exports.queryGlobalView = async (viewName, params, db = null) => { const CreateFuncByName = { - [ViewNames.USER_BY_EMAIL]: exports.createUserEmailView, + [ViewNames.USER_BY_EMAIL]: exports.createNewUserEmailView, [ViewNames.BY_API_KEY]: exports.createApiKeyView, [ViewNames.USER_BY_BUILDERS]: exports.createUserBuildersView, } @@ -98,6 +120,7 @@ exports.queryGlobalView = async (viewName, params, db = null) => { } catch (err) { if (err != null && err.name === "not_found") { const createFunc = CreateFuncByName[viewName] + await removeDeprecated(db, viewName) await createFunc() return exports.queryGlobalView(viewName, params) } else { diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0a17c82873..37804b31a6 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -40,7 +40,7 @@ const env = { DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""), COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, - PLATFORM_URL: process.env.PLATFORM_URL, + PLATFORM_URL: process.env.PLATFORM_URL || "", POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, @@ -54,6 +54,7 @@ const env = { DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", + MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false, DEPLOYMENT_ENVIRONMENT: process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose", _set(key: any, value: any) { diff --git a/packages/backend-core/src/events/processors/PosthogProcessor.ts b/packages/backend-core/src/events/processors/PosthogProcessor.ts index 67407fdd5c..eb12db1dc4 100644 --- a/packages/backend-core/src/events/processors/PosthogProcessor.ts +++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts @@ -2,7 +2,7 @@ import PostHog from "posthog-node" import { Event, Identity, Group, BaseEvent } from "@budibase/types" import { EventProcessor } from "./types" import env from "../../environment" -import context from "../../context" +import * as context from "../../context" const pkg = require("../../../package.json") export default class PosthogProcessor implements EventProcessor { diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index b926334317..2e4ef0da76 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -9,7 +9,7 @@ import { getGlobalDBName, getTenantId, } from "../tenancy" -import context from "../context" +import * as context from "../context" import { DEFINITIONS } from "." import { Migration, diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 77dbc61425..e1f38a798f 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -764,6 +764,11 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/lodash@4.14.180": + version "4.14.180" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.180.tgz#4ab7c9ddfc92ec4a887886483bc14c79fb380670" + integrity sha512-XOKXa1KIxtNXgASAnwj7cnttJxS4fksBRywK/9LzRV5YxrF80BXZIGeQSuoESQ/VkUj30Ae0+YcuHc15wJCB2g== + "@types/mime@^1": version "1.3.2" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index dc6dc27c6f..834ee29c4b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.219-alpha.10", + "version": "1.1.10-alpha.4", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.219-alpha.10", + "@budibase/string-templates": "^1.1.10-alpha.4", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", @@ -66,11 +66,12 @@ "@spectrum-css/radio": "^3.0.2", "@spectrum-css/search": "^3.0.2", "@spectrum-css/sidenav": "^3.0.2", + "@spectrum-css/slider": "3.0.1", "@spectrum-css/statuslight": "^3.0.2", "@spectrum-css/stepper": "^3.0.3", "@spectrum-css/switch": "^1.0.2", "@spectrum-css/table": "^3.0.1", - "@spectrum-css/tabs": "^3.0.1", + "@spectrum-css/tabs": "^3.2.12", "@spectrum-css/tags": "^3.0.2", "@spectrum-css/textfield": "^3.0.1", "@spectrum-css/toast": "^3.0.1", diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 2d23120046..53ba6c7e51 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -82,6 +82,12 @@ .active svg { color: var(--spectrum-global-color-blue-600); } + :global([dir="ltr"] .spectrum-ActionButton .spectrum-Icon) { + margin-left: 0; + } + .is-selected:not(.spectrum-ActionButton--emphasized) { + background: var(--spectrum-global-color-gray-300); + } .noPadding { padding: 0; min-width: 0; diff --git a/packages/bbui/src/Banner/Banner.svelte b/packages/bbui/src/Banner/Banner.svelte index f41fb5f803..3810021a61 100644 --- a/packages/bbui/src/Banner/Banner.svelte +++ b/packages/bbui/src/Banner/Banner.svelte @@ -8,6 +8,7 @@ export let size = "S" export let extraButtonText export let extraButtonAction + export let showCloseButton = true let show = true @@ -39,22 +40,24 @@ {/if} -
- -
+ {#if showCloseButton} +
+ +
+ {/if} {/if} @@ -63,4 +66,7 @@ pointer-events: all; width: 100%; } + .spectrum-Button { + border: 1px solid rgba(255, 255, 255, 0.2); + } diff --git a/packages/bbui/src/Divider/Divider.svelte b/packages/bbui/src/Divider/Divider.svelte index 2b4de9cfb0..e4f0f2fb61 100644 --- a/packages/bbui/src/Divider/Divider.svelte +++ b/packages/bbui/src/Divider/Divider.svelte @@ -16,6 +16,9 @@ /> diff --git a/packages/bbui/src/Form/Core/index.js b/packages/bbui/src/Form/Core/index.js index 96d81855e4..7c81cfd70b 100644 --- a/packages/bbui/src/Form/Core/index.js +++ b/packages/bbui/src/Form/Core/index.js @@ -12,3 +12,4 @@ export { default as CoreDatePicker } from "./DatePicker.svelte" export { default as CoreDropzone } from "./Dropzone.svelte" export { default as CoreStepper } from "./Stepper.svelte" export { default as CoreRichTextField } from "./RichTextField.svelte" +export { default as CoreSlider } from "./Slider.svelte" diff --git a/packages/bbui/src/Form/Search.svelte b/packages/bbui/src/Form/Search.svelte index 25dd98306b..74ffeeb22a 100644 --- a/packages/bbui/src/Form/Search.svelte +++ b/packages/bbui/src/Form/Search.svelte @@ -10,6 +10,7 @@ export let disabled = false export let updateOnChange = true export let quiet = false + export let inputRef const dispatch = createEventDispatcher() const onChange = e => { @@ -25,6 +26,7 @@ {value} {placeholder} {quiet} + bind:inputRef on:change={onChange} on:click on:input diff --git a/packages/bbui/src/Form/Select.svelte b/packages/bbui/src/Form/Select.svelte index 0df27e2ff0..1b68746c5e 100644 --- a/packages/bbui/src/Form/Select.svelte +++ b/packages/bbui/src/Form/Select.svelte @@ -14,6 +14,7 @@ export let getOptionLabel = option => extractProperty(option, "label") export let getOptionValue = option => extractProperty(option, "value") export let getOptionIcon = option => option?.icon + export let getOptionColour = option => option?.colour export let quiet = false export let autoWidth = false export let sort = false @@ -47,6 +48,7 @@ {getOptionLabel} {getOptionValue} {getOptionIcon} + {getOptionColour} on:change={onChange} on:click /> diff --git a/packages/bbui/src/Form/Slider.svelte b/packages/bbui/src/Form/Slider.svelte new file mode 100644 index 0000000000..34b2251b35 --- /dev/null +++ b/packages/bbui/src/Form/Slider.svelte @@ -0,0 +1,24 @@ + + + + + diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index eee1d7fbae..9c99178fdb 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -47,7 +47,7 @@ {#if tooltip && showTooltip}
- +
{/if} diff --git a/packages/bbui/src/IconSideNav/IconSideNav.svelte b/packages/bbui/src/IconSideNav/IconSideNav.svelte new file mode 100644 index 0000000000..e8144402e4 --- /dev/null +++ b/packages/bbui/src/IconSideNav/IconSideNav.svelte @@ -0,0 +1,14 @@ +
+ +
+ + diff --git a/packages/bbui/src/IconSideNav/IconSideNavItem.svelte b/packages/bbui/src/IconSideNav/IconSideNavItem.svelte new file mode 100644 index 0000000000..46625c9707 --- /dev/null +++ b/packages/bbui/src/IconSideNav/IconSideNavItem.svelte @@ -0,0 +1,56 @@ + + +
(showTooltip = true)} + on:focus={() => (showTooltip = true)} + on:mouseleave={() => (showTooltip = false)} + on:click +> + + {#if tooltip && showTooltip} +
+ +
+ {/if} +
+ + diff --git a/packages/bbui/src/Notification/NotificationDisplay.svelte b/packages/bbui/src/Notification/NotificationDisplay.svelte index 0b846f06ce..0f7e93eb23 100644 --- a/packages/bbui/src/Notification/NotificationDisplay.svelte +++ b/packages/bbui/src/Notification/NotificationDisplay.svelte @@ -9,7 +9,7 @@
{#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)} -
+
.notifications { position: fixed; - top: 20px; + bottom: 40px; left: 0; right: 0; margin: 0 auto; diff --git a/packages/bbui/src/StatusLight/StatusLight.svelte b/packages/bbui/src/StatusLight/StatusLight.svelte index f56fee0c2a..a0c72443a6 100644 --- a/packages/bbui/src/StatusLight/StatusLight.svelte +++ b/packages/bbui/src/StatusLight/StatusLight.svelte @@ -17,10 +17,13 @@ export let negative = false export let disabled = false export let active = false + export let color = null
+ + diff --git a/packages/bbui/src/Tabs/Tab.svelte b/packages/bbui/src/Tabs/Tab.svelte index 04791619dc..c25be7dbc9 100644 --- a/packages/bbui/src/Tabs/Tab.svelte +++ b/packages/bbui/src/Tabs/Tab.svelte @@ -79,4 +79,10 @@ .emphasized { color: var(--spectrum-global-color-blue-600); } + .spectrum-Tabs-item { + color: var(--spectrum-global-color-gray-600); + } + .spectrum-Tabs-item.is-selected { + color: var(--spectrum-global-color-gray-900); + } diff --git a/packages/bbui/src/Tabs/Tabs.svelte b/packages/bbui/src/Tabs/Tabs.svelte index 579c61e28d..74edc9cd02 100644 --- a/packages/bbui/src/Tabs/Tabs.svelte +++ b/packages/bbui/src/Tabs/Tabs.svelte @@ -10,8 +10,7 @@ export let noHorizPadding = false export let quiet = false export let emphasized = false - // overlay content from the tab bar onto tabs e.g. for a dropdown - export let onTop = false + export let size = "M" let thisSelected = undefined @@ -74,20 +73,18 @@
{#if $tab.info}
{/if}
@@ -98,26 +95,26 @@ /> diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index 2b16f32b84..7d5baad474 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -64,6 +64,9 @@ export { default as BannerDisplay } from "./Banner/BannerDisplay.svelte" export { default as MarkdownEditor } from "./Markdown/MarkdownEditor.svelte" export { default as MarkdownViewer } from "./Markdown/MarkdownViewer.svelte" export { default as RichTextField } from "./Form/RichTextField.svelte" +export { default as IconSideNav } from "./IconSideNav/IconSideNav.svelte" +export { default as IconSideNavItem } from "./IconSideNav/IconSideNavItem.svelte" +export { default as Slider } from "./Form/Slider.svelte" // Renderers export { default as BoldRenderer } from "./Table/BoldRenderer.svelte" diff --git a/packages/bbui/yarn.lock b/packages/bbui/yarn.lock index 0bff3e86d9..d301afea53 100644 --- a/packages/bbui/yarn.lock +++ b/packages/bbui/yarn.lock @@ -206,6 +206,11 @@ resolved "https://registry.yarnpkg.com/@spectrum-css/sidenav/-/sidenav-3.0.2.tgz#9d70f408d588ee79c69857751010333671f32713" integrity sha512-YpIdH/F0jEICYmoduGrnkTmxwJq1kfKxEp0wOs+ZkQOsvKMv1an7nyhsfOKCQqcGNfYzJ9mJAk7/u5+vsxHa8g== +"@spectrum-css/slider@3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@spectrum-css/slider/-/slider-3.0.1.tgz#5281e6f47eb5a4fd3d1816c138bf66d01d7f2e49" + integrity sha512-DI2dtMRnQuDM1miVzl3SGyR1khUEKnwdXfO5EHDFwkC3yav43F5QogkfjmjFmWWobMVovdJlAuiaaJ/IHejD0Q== + "@spectrum-css/statuslight@^3.0.2": version "3.0.2" resolved "https://registry.yarnpkg.com/@spectrum-css/statuslight/-/statuslight-3.0.2.tgz#dc54b6cd113413dcdb909c486b5d7bae60db65c5" @@ -226,10 +231,10 @@ resolved "https://registry.yarnpkg.com/@spectrum-css/table/-/table-3.0.2.tgz#c666743d569fef81ddc8810fac8cda53b315f8d7" integrity sha512-nt/QNC7NmUank0wozd4FySEX1UIYXuvuOKDyN1II3sxfwFSpJfp/Df9KVMhrYs4EsmB4XMGcoxp8ND/CrvH3ow== -"@spectrum-css/tabs@^3.0.1": - version "3.0.2" - resolved "https://registry.yarnpkg.com/@spectrum-css/tabs/-/tabs-3.0.2.tgz#822316672e7b0dfba66faa988e638ddae18c700e" - integrity sha512-4RNcmwf0wxLpB7M54H02owlj0mKE8neL1+lytQpxOOhlwTO5zdsD82zjvx9tIc8tRnRKuhCCCwTuBxHYstnBmw== +"@spectrum-css/tabs@^3.2.12": + version "3.2.12" + resolved "https://registry.yarnpkg.com/@spectrum-css/tabs/-/tabs-3.2.12.tgz#9b08f23d5aa881b3441af7757800c7173e5685ff" + integrity sha512-rPFUW9SSW4+3/UJ3UrtY2/l3sQvlqB1fqxHLPDjgykvbfrnMejcCTNV4ZrFNHXpE/6+kGnk+yVViSPtWGwJzkA== "@spectrum-css/tags@^3.0.2": version "3.0.2" diff --git a/packages/builder/cypress/integration/addRadioButtons.spec.js b/packages/builder/cypress/integration/addRadioButtons.spec.js index 402db195aa..578b519341 100644 --- a/packages/builder/cypress/integration/addRadioButtons.spec.js +++ b/packages/builder/cypress/integration/addRadioButtons.spec.js @@ -10,6 +10,7 @@ filterTests(['all'], () => { it("should add Radio Buttons options picker on form, add data, and confirm", () => { cy.navigateToFrontend() + cy.wait(500) cy.addComponent("Form", "Form") cy.addComponent("Form", "Options Picker").then((componentId) => { // Provide field setting diff --git a/packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js b/packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js index 69f3c2ec1d..562e1e149f 100644 --- a/packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js +++ b/packages/builder/cypress/integration/adminAndManagement/userManagement.spec.js @@ -43,7 +43,7 @@ filterTests(["smoke", "all"], () => { const uuid = () => Cypress._.random(0, 1e6) const name = uuid() if(i < 1){ - cy.createApp(name) + cy.createApp(name, false) } else { cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000}) cy.wait(1000) @@ -56,7 +56,7 @@ filterTests(["smoke", "all"], () => { // Navigate back to the user cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000}) cy.get(interact.SPECTRUM_SIDENAV).contains("Users").click() - cy.get(interact.SPECTRUM_TABLE, { timeout: 500 }).contains("bbuser").click() + cy.get(interact.SPECTRUM_TABLE, { timeout: 1000 }).contains("bbuser").click() for (let i = 0; i < 3; i++) { cy.get(interact.SPECTRUM_TABLE, { timeout: 3000}) .eq(1) @@ -65,24 +65,24 @@ filterTests(["smoke", "all"], () => { .find(interact.SPECTRUM_TABLE_CELL) .eq(0) .click() - cy.get(interact.SPECTRUM_DIALOG_GRID, { timeout: 500 }) + cy.get(interact.SPECTRUM_DIALOG_GRID, { timeout: 1000 }) .contains("Choose an option") .click() .then(() => { if (i == 0) { - cy.get(interact.SPECTRUM_MENU, { timeout: 1000 }).contains("Admin").click({ force: true }) + cy.get(interact.SPECTRUM_MENU, { timeout: 2000 }).contains("Admin").click({ force: true }) } else if (i == 1) { - cy.get(interact.SPECTRUM_MENU, { timeout: 1000 }).contains("Power").click({ force: true }) + cy.get(interact.SPECTRUM_MENU, { timeout: 2000 }).contains("Power").click({ force: true }) } else if (i == 2) { - cy.get(interact.SPECTRUM_MENU, { timeout: 1000 }).contains("Basic").click({ force: true }) + cy.get(interact.SPECTRUM_MENU, { timeout: 2000 }).contains("Basic").click({ force: true }) } - cy.get(interact.SPECTRUM_BUTTON, { timeout: 1000 }) + cy.get(interact.SPECTRUM_BUTTON, { timeout: 2000 }) .contains("Update role") .click({ force: true }) }) - cy.reload() + cy.reload({ timeout: 5000 }) cy.wait(1000) } // Confirm roles exist within Configure roles table diff --git a/packages/builder/cypress/integration/appOverview.spec.js b/packages/builder/cypress/integration/appOverview.spec.js index 5856786bc4..dbfce3ce63 100644 --- a/packages/builder/cypress/integration/appOverview.spec.js +++ b/packages/builder/cypress/integration/appOverview.spec.js @@ -6,7 +6,7 @@ filterTests(["all"], () => { before(() => { cy.login() cy.deleteAllApps() - cy.createTestApp() + cy.createApp("Cypress Tests") }) it("Should be accessible from the applications list", () => { @@ -82,13 +82,14 @@ filterTests(["all"], () => { }) it("Should reflect the app deployment state", () => { - cy.visit(`${Cypress.config().baseUrl}/builder`) + cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 }) cy.get(".appTable .app-row-actions button") .contains("Edit") .eq(0) .click({ force: true }) - cy.get(".toprightnav button.spectrum-Button") + cy.wait(500) + cy.get(".toprightnav button.spectrum-Button", { timeout: 2000 }) .contains("Publish") .click({ force: true }) cy.get(".spectrum-Modal [data-cy='deploy-app-modal']") diff --git a/packages/builder/cypress/integration/createView.spec.js b/packages/builder/cypress/integration/createView.spec.js index a2d09d97bf..9adc486f70 100644 --- a/packages/builder/cypress/integration/createView.spec.js +++ b/packages/builder/cypress/integration/createView.spec.js @@ -5,7 +5,6 @@ filterTests(['smoke', 'all'], () => { context("Create a View", () => { before(() => { cy.login() - cy.createTestApp() cy.createTable("data") cy.addColumn("data", "group", "Text") diff --git a/packages/builder/cypress/integration/datasources/postgreSql.spec.js b/packages/builder/cypress/integration/datasources/postgreSql.spec.js index 598055225e..ccecfbd5df 100644 --- a/packages/builder/cypress/integration/datasources/postgreSql.spec.js +++ b/packages/builder/cypress/integration/datasources/postgreSql.spec.js @@ -217,24 +217,24 @@ filterTests(["all"], () => { it("should edit a query name", () => { // Access query - cy.get(".hierarchy-items-container") + cy.get(".hierarchy-items-container", { timeout: 2000 }) .contains(queryName + " (1)") .click() // Rename query - cy.get(".spectrum-Form-item") + cy.wait(1000) + cy.get(".spectrum-Form-item", { timeout: 2000 }) .eq(0) .within(() => { cy.get("input").clear().type(queryRename) }) // Run and Save query - cy.get(".spectrum-Button").contains("Run Query").click({ force: true }) - cy.wait(500) - cy.get(".spectrum-Button", { timeout: 500 }).contains("Save Query").click({ force: true }) - //cy.reload() - //cy.wait(500) - cy.get(".nav-item").should("contain", queryRename) + cy.get(".spectrum-Button", { timeout: 2000 }).contains("Run Query").click({ force: true }) + cy.wait(1000) + cy.get(".spectrum-Button", { timeout: 2000 }).contains("Save Query").click({ force: true }) + cy.reload({ timeout: 5000 }) + cy.get(".nav-item", { timeout: 2000 }).should("contain", queryRename) }) it("should delete a query", () => { @@ -251,6 +251,7 @@ filterTests(["all"], () => { .contains("Delete Query") .click({ force: true }) // Confirm deletion + cy.reload({ timeout: 5000 }) cy.get(".nav-item", { timeout: 1000 }).should("not.contain", queryName) }) diff --git a/packages/builder/cypress/support/commands.js b/packages/builder/cypress/support/commands.js index 16bb11ea8e..b2ab8e678c 100644 --- a/packages/builder/cypress/support/commands.js +++ b/packages/builder/cypress/support/commands.js @@ -134,16 +134,18 @@ Cypress.Commands.add("createApp", (name, addDefaultTable) => { const shouldCreateDefaultTable = typeof addDefaultTable != "boolean" ? true : addDefaultTable - cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 }) + cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 10000 }) cy.wait(1000) - cy.get(`[data-cy="create-app-btn"]`, { timeout: 2000 }).click({ force: true }) + cy.get(`[data-cy="create-app-btn"]`, { timeout: 5000 }).click({ force: true }) // If apps already exist cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`) .its("body") .then(val => { if (val.length > 0) { - cy.get(`[data-cy="create-app-btn"]`).click({ force: true }) + cy.get(`[data-cy="create-app-btn"]`, { timeout: 5000 }).click({ + force: true, + }) } }) @@ -401,17 +403,19 @@ Cypress.Commands.add("createAppFromScratch", appName => { Cypress.Commands.add("createTable", (tableName, initialTable) => { if (!initialTable) { cy.navigateToDataSection() - cy.get(`[data-cy="new-table"]`).click() + cy.get(`[data-cy="new-table"]`, { timeout: 2000 }).click() } cy.wait(2000) - cy.get(".item") + cy.get(".item", { timeout: 2000 }) .contains("Budibase DB") .click({ force: true }) .then(() => { - cy.get(".spectrum-Button").contains("Continue").click({ force: true }) + cy.get(".spectrum-Button", { timeout: 2000 }) + .contains("Continue") + .click({ force: true }) }) - cy.get(".spectrum-Modal").within(() => { - cy.get("input", { timeout: 1000 }).first().type(tableName).blur() + cy.get(".spectrum-Modal", { timeout: 2000 }).within(() => { + cy.get("input", { timeout: 2000 }).first().type(tableName).blur() cy.get(".spectrum-ButtonGroup").contains("Create").click() }) cy.contains(tableName).should("be.visible") @@ -505,12 +509,13 @@ Cypress.Commands.add("addCustomSourceOptions", totalOptions => { // DESIGN AREA Cypress.Commands.add("addComponent", (category, component) => { if (category) { - cy.get(`[data-cy="category-${category}"]`, { timeout: 1000 }).click({ + cy.get(`[data-cy="category-${category}"]`, { timeout: 3000 }).click({ force: true, }) } + cy.wait(500) if (component) { - cy.get(`[data-cy="component-${component}"]`, { timeout: 1000 }).click({ + cy.get(`[data-cy="component-${component}"]`, { timeout: 3000 }).click({ force: true, }) } @@ -518,7 +523,7 @@ Cypress.Commands.add("addComponent", (category, component) => { cy.location().then(loc => { const params = loc.pathname.split("/") const componentId = params[params.length - 1] - cy.getComponent(componentId).should("exist") + cy.getComponent(componentId, { timeout: 3000 }).should("exist") return cy.wrap(componentId) }) }) @@ -622,8 +627,8 @@ Cypress.Commands.add("navigateToFrontend", () => { // Clicks on Design tab and then the Home nav item cy.wait(500) cy.contains("Design").click() - cy.get(".spectrum-Search").type("/") - cy.get(".nav-item").contains("home").click() + cy.get(".spectrum-Search", { timeout: 2000 }).type("/") + cy.get(".nav-item", { timeout: 2000 }).contains("home").click() }) Cypress.Commands.add("navigateToDataSection", () => { @@ -783,7 +788,7 @@ Cypress.Commands.add("createRestQuery", (method, restUrl, queryPrettyName) => { // MISC Cypress.Commands.add("closeModal", () => { - cy.get(".spectrum-Modal").within(() => { + cy.get(".spectrum-Modal", { timeout: 2000 }).within(() => { cy.get(".close-icon").click() cy.wait(1000) // Wait for modal to close }) diff --git a/packages/builder/package.json b/packages/builder/package.json index ab580440ad..cd86c2414d 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.219-alpha.10", + "version": "1.1.10-alpha.4", "license": "GPL-3.0", "private": true, "scripts": { @@ -69,10 +69,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.219-alpha.10", - "@budibase/client": "^1.0.219-alpha.10", - "@budibase/frontend-core": "^1.0.219-alpha.10", - "@budibase/string-templates": "^1.0.219-alpha.10", + "@budibase/bbui": "^1.1.10-alpha.4", + "@budibase/client": "^1.1.10-alpha.4", + "@budibase/frontend-core": "^1.1.10-alpha.4", + "@budibase/string-templates": "^1.1.10-alpha.4", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", @@ -95,7 +95,7 @@ "@babel/preset-env": "^7.13.12", "@babel/runtime": "^7.13.10", "@rollup/plugin-replace": "^2.4.2", - "@roxi/routify": "2.18.0", + "@roxi/routify": "2.18.5", "@sveltejs/vite-plugin-svelte": "1.0.0-next.19", "@testing-library/jest-dom": "^5.11.10", "@testing-library/svelte": "^3.0.0", @@ -113,7 +113,7 @@ "rollup": "^2.44.0", "rollup-plugin-copy": "^3.4.0", "start-server-and-test": "^1.12.1", - "svelte": "^3.38.2", + "svelte": "^3.49.0", "svelte-jester": "^1.3.2", "ts-node": "^10.4.0", "tsconfig-paths": "4.0.0", diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index 234f83d7cc..bebd06c6d7 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -20,7 +20,7 @@ import { } from "@budibase/string-templates" import { TableNames } from "../constants" import { JSONUtils } from "@budibase/frontend-core" -import ActionDefinitions from "components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/manifest.json" +import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json" // Regex to match all instances of template strings const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g @@ -478,11 +478,17 @@ const getUrlBindings = asset => { } }) const safeURL = makePropSafe("url") - return params.map(param => ({ + const urlParamBindings = params.map(param => ({ type: "context", runtimeBinding: `${safeURL}.${makePropSafe(param)}`, readableBinding: `URL.${param}`, })) + const queryParamsBinding = { + type: "context", + runtimeBinding: makePropSafe("query"), + readableBinding: "Query params", + } + return urlParamBindings.concat([queryParamsBinding]) } const getRoleBindings = () => { @@ -782,6 +788,13 @@ export const getAllStateVariables = () => { }) }) + // Add on load settings from screens + get(store).screens.forEach(screen => { + if (screen.onLoad) { + eventSettings.push(screen.onLoad) + } + }) + // Extract all state keys from any "update state" actions in each setting let bindingSet = new Set() eventSettings.forEach(setting => { diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index 619bdd94a1..28ef1f4376 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -1,65 +1,77 @@ import { getFrontendStore } from "./store/frontend" import { getAutomationStore } from "./store/automation" import { getThemeStore } from "./store/theme" -import { derived, writable } from "svelte/store" -import { FrontendTypes, LAYOUT_NAMES } from "../constants" +import { derived } from "svelte/store" +import { LAYOUT_NAMES } from "../constants" import { findComponent, findComponentPath } from "./componentUtils" +import { RoleUtils } from "@budibase/frontend-core" export const store = getFrontendStore() export const automationStore = getAutomationStore() export const themeStore = getThemeStore() -export const currentAsset = derived(store, $store => { - const type = $store.currentFrontEndType - if (type === FrontendTypes.SCREEN) { - return $store.screens.find(screen => screen._id === $store.selectedScreenId) - } else if (type === FrontendTypes.LAYOUT) { - return $store.layouts.find(layout => layout._id === $store.selectedLayoutId) - } - return null +export const selectedScreen = derived(store, $store => { + return $store.screens.find(screen => screen._id === $store.selectedScreenId) +}) + +export const selectedLayout = derived(store, $store => { + return $store.layouts?.find(layout => layout._id === $store.selectedLayoutId) }) export const selectedComponent = derived( - [store, currentAsset], - ([$store, $currentAsset]) => { - if (!$currentAsset || !$store.selectedComponentId) { + [store, selectedScreen], + ([$store, $selectedScreen]) => { + if (!$selectedScreen || !$store.selectedComponentId) { return null } - return findComponent($currentAsset?.props, $store.selectedComponentId) + return findComponent($selectedScreen?.props, $store.selectedComponentId) } ) +export const sortedScreens = derived(store, $store => { + return $store.screens.slice().sort((a, b) => { + // Sort by role first + const roleA = RoleUtils.getRolePriority(a.routing.roleId) + const roleB = RoleUtils.getRolePriority(b.routing.roleId) + if (roleA !== roleB) { + return roleA > roleB ? -1 : 1 + } + // Then put home screens first + const homeA = !!a.routing.homeScreen + const homeB = !!b.routing.homeScreen + if (homeA !== homeB) { + return homeA ? -1 : 1 + } + // Then sort alphabetically by each URL param + const aParams = a.routing.route.split("/") + const bParams = b.routing.route.split("/") + let minParams = Math.min(aParams.length, bParams.length) + for (let i = 0; i < minParams; i++) { + if (aParams[i] === bParams[i]) { + continue + } + return aParams[i] < bParams[i] ? -1 : 1 + } + // Then sort by the fewest amount of URL params + return aParams.length < bParams.length ? -1 : 1 + }) +}) + export const selectedComponentPath = derived( - [store, currentAsset], - ([$store, $currentAsset]) => { + [store, selectedScreen], + ([$store, $selectedScreen]) => { return findComponentPath( - $currentAsset?.props, + $selectedScreen?.props, $store.selectedComponentId ).map(component => component._id) } ) -export const currentAssetId = derived(store, $store => { - return $store.currentFrontEndType === FrontendTypes.SCREEN - ? $store.selectedScreenId - : $store.selectedLayoutId -}) - -export const currentAssetName = derived(currentAsset, $currentAsset => { - return $currentAsset?.name -}) - -// leave this as before for consistency -export const allScreens = derived(store, $store => { - return $store.screens -}) - export const mainLayout = derived(store, $store => { return $store.layouts?.find( layout => layout._id === LAYOUT_NAMES.MASTER.PRIVATE ) }) -export const selectedAccessRole = writable("BASIC") - -export const screenSearchString = writable(null) +// For compatibility +export const currentAsset = selectedScreen diff --git a/packages/builder/src/builderStore/store/automation/index.js b/packages/builder/src/builderStore/store/automation/index.js index dd09e3356a..af102ab694 100644 --- a/packages/builder/src/builderStore/store/automation/index.js +++ b/packages/builder/src/builderStore/store/automation/index.js @@ -68,7 +68,19 @@ const automationActions = store => ({ return state }) }, - + duplicate: async automation => { + const response = await API.createAutomation({ + ...automation, + name: `${automation.name} - copy`, + _id: undefined, + _ref: undefined, + }) + store.update(state => { + state.automations = [...state.automations, response.automation] + store.actions.select(response.automation) + return state + }) + }, save: async automation => { const response = await API.updateAutomation(automation) store.update(state => { diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index ec810e5c31..16ae5ce215 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -1,12 +1,6 @@ import { get, writable } from "svelte/store" import { cloneDeep } from "lodash/fp" -import { - allScreens, - currentAsset, - mainLayout, - selectedComponent, - selectedAccessRole, -} from "builderStore" +import { currentAsset, mainLayout, selectedComponent } from "builderStore" import { datasources, integrations, @@ -15,7 +9,6 @@ import { tables, } from "stores/backend" import { API } from "api" -import { FrontendTypes } from "constants" import analytics, { Events } from "analytics" import { findComponentType, @@ -27,6 +20,7 @@ import { makeComponentUnique, } from "../componentUtils" import { Helpers } from "@budibase/bbui" +import { DefaultAppTheme, LAYOUT_NAMES } from "../../constants" const INITIAL_FRONTEND_STATE = { apps: [], @@ -47,10 +41,6 @@ const INITIAL_FRONTEND_STATE = { messagePassing: false, continueIfAction: false, }, - currentFrontEndType: "none", - selectedScreenId: "", - selectedLayoutId: "", - selectedComponentId: "", errors: [], hasAppPackage: false, libraries: null, @@ -61,6 +51,11 @@ const INITIAL_FRONTEND_STATE = { customTheme: {}, previewDevice: "desktop", highlightedSettingKey: null, + + // URL params + selectedScreenId: null, + selectedComponentId: null, + selectedLayoutId: null, } export const getFrontendStore = () => { @@ -100,6 +95,7 @@ export const getFrontendStore = () => { previousTopNavPath: {}, version: application.version, revertableVersion: application.revertableVersion, + navigation: application.navigation || {}, })) // Initialise backend stores @@ -108,6 +104,35 @@ export const getFrontendStore = () => { await integrations.init() await queries.init() await tables.init() + + // Add navigation settings to old apps + if (!application.navigation) { + const layout = layouts.find(x => x._id === LAYOUT_NAMES.MASTER.PRIVATE) + const customTheme = application.customTheme + let navigationSettings = { + navigation: "Top", + title: application.name, + navWidth: "Large", + navBackground: + customTheme?.navBackground || DefaultAppTheme.navBackground, + navTextColor: + customTheme?.navTextColor || DefaultAppTheme.navTextColor, + } + if (layout) { + navigationSettings.hideLogo = layout.props.hideLogo + navigationSettings.hideTitle = layout.props.hideTitle + navigationSettings.title = layout.props.title || application.name + navigationSettings.logoUrl = layout.props.logoUrl + navigationSettings.links = layout.props.links + navigationSettings.navigation = layout.props.navigation || "Top" + navigationSettings.sticky = layout.props.sticky + navigationSettings.navWidth = layout.props.width || "Large" + if (navigationSettings.navigation === "None") { + navigationSettings.navigation = "Top" + } + } + await store.actions.navigation.save(navigationSettings) + } }, theme: { save: async theme => { @@ -135,6 +160,19 @@ export const getFrontendStore = () => { }) }, }, + navigation: { + save: async navigation => { + const appId = get(store).appId + await API.saveAppMetadata({ + appId, + metadata: { navigation }, + }) + store.update(state => { + state.navigation = navigation + return state + }) + }, + }, routing: { fetch: async () => { const response = await API.fetchAppRoutes() @@ -147,18 +185,12 @@ export const getFrontendStore = () => { screens: { select: screenId => { store.update(state => { - let screens = get(allScreens) + let screens = state.screens let screen = screens.find(screen => screen._id === screenId) || screens[0] if (!screen) return state - // Update role to the screen's role setting so that it will always - // be visible - selectedAccessRole.set(screen.routing.roleId) - - state.currentFrontEndType = FrontendTypes.SCREEN state.selectedScreenId = screen._id - state.currentView = "detail" state.selectedComponentId = screen.props?._id return state }) @@ -221,16 +253,44 @@ export const getFrontendStore = () => { // Refresh routes await store.actions.routing.fetch() }, + updateHomeScreen: async (screen, makeHomeScreen = true) => { + let promises = [] + + // Find any existing home screen for this role so we can remove it, + // if we are setting this to be the new home screen + if (makeHomeScreen) { + const roleId = screen.routing.roleId + let existingHomeScreen = get(store).screens.find(s => { + return ( + s.routing.roleId === roleId && + s.routing.homeScreen && + s._id !== screen._id + ) + }) + if (existingHomeScreen) { + existingHomeScreen.routing.homeScreen = false + promises.push(store.actions.screens.save(existingHomeScreen)) + } + } + + // Update the passed in screen + screen.routing.homeScreen = makeHomeScreen + promises.push(store.actions.screens.save(screen)) + return await Promise.all(promises) + }, + removeCustomLayout: async screen => { + // Pull relevant settings from old layout, if required + const layout = get(store).layouts.find(x => x._id === screen.layoutId) + screen.layoutId = null + screen.showNavigation = layout?.props.navigation !== "None" + screen.width = layout?.props.width || "Large" + await store.actions.screens.save(screen) + }, }, preview: { saveSelected: async () => { - const state = get(store) const selectedAsset = get(currentAsset) - if (state.currentFrontEndType !== FrontendTypes.LAYOUT) { - return await store.actions.screens.save(selectedAsset) - } else { - return await store.actions.layouts.save(selectedAsset) - } + return await store.actions.screens.save(selectedAsset) }, setDevice: device => { store.update(state => { @@ -245,8 +305,6 @@ export const getFrontendStore = () => { const layout = store.actions.layouts.find(layoutId) || get(store).layouts[0] if (!layout) return - state.currentFrontEndType = FrontendTypes.LAYOUT - state.currentView = "detail" state.selectedLayoutId = layout._id state.selectedComponentId = layout.props?._id return state @@ -297,32 +355,6 @@ export const getFrontendStore = () => { }, }, components: { - select: component => { - const asset = get(currentAsset) - if (!asset || !component) { - return - } - - // If this is the root component, select the asset instead - const parent = findComponentParent(asset.props, component._id) - if (parent == null) { - const state = get(store) - const isLayout = state.currentFrontEndType === FrontendTypes.LAYOUT - if (isLayout) { - store.actions.layouts.select(asset._id) - } else { - store.actions.screens.select(asset._id) - } - return - } - - // Otherwise select the component - store.update(state => { - state.selectedComponentId = component._id - state.currentView = "component" - return state - }) - }, getDefinition: componentName => { if (!componentName) { return null @@ -418,7 +450,6 @@ export const getFrontendStore = () => { // Save components and update UI await store.actions.preview.saveSelected() store.update(state => { - state.currentView = "component" state.selectedComponentId = componentInstance._id return state }) @@ -461,11 +492,14 @@ export const getFrontendStore = () => { parent._children = parent._children.filter( child => child._id !== component._id ) - store.actions.components.select(parent) + store.update(state => { + state.selectedComponentId = parent._id + return state + }) } await store.actions.preview.saveSelected() }, - copy: (component, cut = false) => { + copy: (component, cut = false, selectParent = true) => { const selectedAsset = get(currentAsset) if (!selectedAsset) { return null @@ -485,7 +519,12 @@ export const getFrontendStore = () => { parent._children = parent._children.filter( child => child._id !== component._id ) - store.actions.components.select(parent) + if (selectParent) { + store.update(state => { + state.selectedComponentId = parent._id + return state + }) + } } } }, @@ -536,7 +575,7 @@ export const getFrontendStore = () => { // Save and select the new component promises.push(store.actions.preview.saveSelected()) - store.actions.components.select(componentToPaste) + state.selectedComponentId = componentToPaste._id return state }) await Promise.all(promises) @@ -578,35 +617,38 @@ export const getFrontendStore = () => { }, links: { save: async (url, title) => { - const layout = get(mainLayout) - if (!layout) { + const navigation = get(store).navigation + let links = [...navigation?.links] + + // Skip if we have an identical link + if (links.find(link => link.url === url && link.text === title)) { return } - // Add link setting to main layout - if (!layout.props.links) { - layout.props.links = [] - } - layout.props.links.push({ + links.push({ text: title, url, }) - - await store.actions.layouts.save(layout) + await store.actions.navigation.save({ + ...navigation, + links: [...links], + }) }, delete: async urls => { - const layout = get(mainLayout) - if (!layout?.props.links?.length) { + const navigation = get(store).navigation + let links = navigation?.links + if (!links?.length) { return } // Filter out the URLs to delete urls = Array.isArray(urls) ? urls : [urls] - layout.props.links = layout.props.links.filter( - link => !urls.includes(link.url) - ) + links = links.filter(link => !urls.includes(link.url)) - await store.actions.layouts.save(layout) + await store.actions.navigation.save({ + ...navigation, + links, + }) }, }, settings: { diff --git a/packages/builder/src/builderStore/store/screenTemplates/utils/Screen.js b/packages/builder/src/builderStore/store/screenTemplates/utils/Screen.js index 272f627163..6fc79e53b0 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/utils/Screen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/utils/Screen.js @@ -5,7 +5,8 @@ export class Screen extends BaseStructure { constructor() { super(true) this._json = { - layoutId: "layout_private_master", + showNavigation: true, + width: "Large", props: { _id: Helpers.uuid(), _component: "@budibase/standard-components/container", @@ -26,6 +27,7 @@ export class Screen extends BaseStructure { routing: { route: "", roleId: "BASIC", + homeScreen: false, }, name: "screen-id", } diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index e8b61d7402..99c6f251f9 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -96,7 +96,7 @@ onSelect(block) }} > - +
diff --git a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte index a2eb904c94..c6585b0bce 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte @@ -67,27 +67,20 @@ {/if}
- + -
-