diff --git a/.dockerignore b/.dockerignore index 92bd33894e..86e1f739f7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,14 @@ -packages/server/node_modules -packages/builder -packages/frontend-core -packages/backend-core -packages/worker/node_modules -packages/cli -packages/client -packages/bbui -packages/string-templates +* +!/packages/ +!/scripts/ +/packages/*/node_modules +packages/server/scripts/ +!packages/server/scripts/integrations/oracle +!nx.json +!/hosting/single/ +!/hosting/letsencrypt / + +!package.json +!yarn.lock +!lerna.json +!.yarnrc \ No newline at end of file diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index c8bc31f41d..840d580892 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -10,7 +10,6 @@ on: push: branches: - master - - develop pull_request: workflow_dispatch: @@ -262,11 +261,7 @@ jobs: branch="${{ github.base_ref || github.ref_name }}" echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" - if [[ $branch == "master" ]]; then - base_commit=$(git rev-parse origin/master) - elif [[ $branch == "develop" ]]; then - base_commit=$(git rev-parse origin/develop) - fi + base_commit=$(git rev-parse origin/master) if [[ ! -z $base_commit ]]; then echo "target_branch=$branch" diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 0ec3b43598..46cb781730 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -4,7 +4,13 @@ on: pull_request: types: [closed] branches: - - develop + - master + workflow_dispatch: + inputs: + BRANCH: + type: string + description: Which featurebranch branch to destroy? + required: true jobs: release: @@ -13,8 +19,8 @@ jobs: - uses: actions/checkout@v3 - uses: passeidireto/trigger-external-workflow-action@main env: - PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.ref }} + PAYLOAD_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.BRANCH || github.head_ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} with: repository: budibase/budibase-deploys event: featurebranch-qa-close diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 2c6302b56a..f1fb12c087 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -3,7 +3,6 @@ name: deploy-featurebranch on: pull_request: branches: - - develop - master jobs: diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml deleted file mode 100644 index 9b7bca4770..0000000000 --- a/.github/workflows/deploy-preprod.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "deploy-preprod" -on: - workflow_dispatch: - workflow_call: - -jobs: - deploy-to-legacy-preprod-env: - runs-on: ubuntu-latest - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-preprod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} - diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml deleted file mode 100644 index bd727b7865..0000000000 --- a/.github/workflows/release-develop.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Budibase Prerelease -concurrency: - group: release-prerelease - cancel-in-progress: false - -on: - push: - tags: - - "*-alpha.*" - workflow_dispatch: - -env: - # Posthog token used by ui at build time - # disable unless needed for testing - # POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - FEATURE_PREVIEW_URL: https://budirelease.live - -jobs: - release-images: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not develop - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then - echo "Tag is not in develop" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. - git config --global user.name "Budibase Staging Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release:develop - - - name: Build/release Docker images - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - yarn build:docker:develop - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: develop" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get the current budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-qa-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index b4991cbfbe..3af3a751ad 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -110,19 +110,13 @@ jobs: git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" git push - deploy-to-legacy-preprod-env: - needs: [release-images] - uses: ./.github/workflows/deploy-preprod.yml - secrets: inherit - # Trigger deploy to new EKS preprod environment - trigger-deploy-to-preprod-env: + trigger-deploy-to-qa-env: needs: [release-helm-chart] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: Get the latest budibase release version + - name: Get the current budibase release version id: version run: | release_version=$(cat lerna.json | jq -r '.version') @@ -133,5 +127,5 @@ jobs: PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} with: repository: budibase/budibase-deploys - event: budicloud-preprod-deploy + event: budicloud-qa-deploy github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-singleimage-test.yml b/.github/workflows/release-singleimage-test.yml new file mode 100644 index 0000000000..79b9afdd44 --- /dev/null +++ b/.github/workflows/release-singleimage-test.yml @@ -0,0 +1,69 @@ +name: Test + +on: + workflow_dispatch: + +env: + CI: true + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + REGISTRY_URL: registry.hub.docker.com + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} +jobs: + build: + name: "build" + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x] + steps: + - name: "Checkout" + uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + - name: Run Yarn + run: yarn + - name: Run Yarn Build + run: yarn build --scope @budibase/server --scope @budibase/worker + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_API_KEY }} + - name: Get the latest release version + id: version + run: | + release_version=$(cat lerna.json | jq -r '.version') + echo $release_version + echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV + - name: Tag and release Budibase service docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + pull: true + platforms: linux/amd64,linux/arm64 + tags: budibase/budibase-test:test + file: ./hosting/single/Dockerfile.v2 + cache-from: type=registry,ref=budibase/budibase-test:test + cache-to: type=inline + - name: Tag and release Budibase Azure App Service docker image + uses: docker/build-push-action@v2 + with: + context: . + push: true + platforms: linux/amd64 + build-args: TARGETBUILD=aas + tags: budibase/budibase-test:aas + file: ./hosting/single/Dockerfile.v2 diff --git a/.github/workflows/tag-prerelease.yml b/.github/workflows/tag-prerelease.yml deleted file mode 100644 index f6446c55f5..0000000000 --- a/.github/workflows/tag-prerelease.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tag prerelease -concurrency: - group: tag-prerelease - cancel-in-progress: false - -on: - push: - branches: - - develop - paths: - - ".aws/**" - - ".github/**" - - "charts/**" - - "packages/**" - - "scripts/**" - - "package.json" - - "yarn.lock" - workflow_dispatch: - -jobs: - tag-prerelease: - runs-on: ubuntu-latest - - steps: - - name: Fail if branch is not develop - if: github.ref != 'refs/heads/develop' - run: | - echo "Ref is not develop, you must run this job from develop." - exit 1 - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - run: cd scripts && yarn - - name: Tag prerelease - run: | - cd scripts - # setup the username and email. - git config --global user.name "Budibase Staging Release Bot" - git config --global user.email "<>" - ./versionCommit.sh prerelease diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index 191c3ad9ef..eaf71ae61a 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -4,17 +4,6 @@ concurrency: cancel-in-progress: false on: - push: - branches: - - master - paths: - - ".aws/**" - - ".github/**" - - "charts/**" - - "packages/**" - - "scripts/**" - - "package.json" - - "yarn.lock" workflow_dispatch: inputs: versioning: diff --git a/.yarnrc b/.yarnrc index 21fa517e23..a3a3d23ec6 100644 --- a/.yarnrc +++ b/.yarnrc @@ -1 +1 @@ -network-timeout 100000 +network-timeout 1000000 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 3a32075a33..77afd9453b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -138,6 +138,8 @@ To develop the Budibase platform you'll need [Docker](https://www.docker.com/) a `yarn setup` will check that all necessary components are installed and setup the repo for usage. +If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above command. + ##### Manual method The following commands can be executed to manually get Budibase up and running (assuming Docker/Docker Compose has been installed). @@ -146,6 +148,8 @@ The following commands can be executed to manually get Budibase up and running ( `yarn build` will build all budibase packages. +If you have access to the `@budibase/pro` submodule then please follow the Pro section of this guide before running the above commands. + #### 4. Running To run the budibase server and builder in dev mode (i.e. with live reloading): diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 new file mode 100644 index 0000000000..b1abe6d53e --- /dev/null +++ b/hosting/single/Dockerfile.v2 @@ -0,0 +1,126 @@ +FROM node:18-slim as build + +# install node-gyp dependencies +RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq + + +# copy and install dependencies +WORKDIR /app +COPY package.json . +COPY yarn.lock . +COPY lerna.json . +COPY .yarnrc . + +COPY packages/server/package.json packages/server/package.json +COPY packages/worker/package.json packages/worker/package.json +# string-templates does not get bundled during the esbuild process, so we want to use the local version +COPY packages/string-templates/package.json packages/string-templates/package.json + + +COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh +RUN chmod +x ./scripts/removeWorkspaceDependencies.sh +RUN ./scripts/removeWorkspaceDependencies.sh + + +# We will never want to sync pro, but the script is still required +RUN echo '' > scripts/syncProPackage.js +RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production + +# copy the actual code +COPY packages/server/dist packages/server/dist +COPY packages/server/pm2.config.js packages/server/pm2.config.js +COPY packages/server/client packages/server/client +COPY packages/server/builder packages/server/builder +COPY packages/worker/dist packages/worker/dist +COPY packages/worker/pm2.config.js packages/worker/pm2.config.js +COPY packages/string-templates packages/string-templates + + +FROM budibase/couchdb as runner +ARG TARGETARCH +ENV TARGETARCH $TARGETARCH +#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) +# e.g. docker build --build-arg TARGETBUILD=aas .... +ARG TARGETBUILD=single +ENV TARGETBUILD $TARGETBUILD + +# install base dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + +# Install postgres client for pg_dump utils +RUN apt install software-properties-common apt-transport-https gpg -y \ + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https gpg -y + +# install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx +WORKDIR /nodejs +RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ + bash /tmp/nodesource_setup.sh && \ + apt-get install -y --no-install-recommends libaio1 nodejs && \ + npm install --global yarn pm2 + +# setup nginx +COPY hosting/single/nginx/nginx.conf /etc/nginx +COPY hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default +RUN mkdir -p /var/log/nginx && \ + touch /var/log/nginx/error.log && \ + touch /var/run/nginx.pid && \ + usermod -a -G tty www-data + +WORKDIR / +RUN mkdir -p scripts/integrations/oracle +COPY packages/server/scripts/integrations/oracle scripts/integrations/oracle +RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh + +# setup minio +WORKDIR /minio +COPY scripts/install-minio.sh ./install.sh +RUN chmod +x install.sh && ./install.sh + +# setup runner file +WORKDIR / +COPY hosting/single/runner.sh . +RUN chmod +x ./runner.sh +COPY hosting/single/healthcheck.sh . +RUN chmod +x ./healthcheck.sh + +# Script below sets the path for storing data based on $DATA_DIR +# For Azure App Service install SSH & point data locations to /home +COPY hosting/single/ssh/sshd_config /etc/ +COPY hosting/single/ssh/ssh_setup.sh /tmp +RUN /build-target-paths.sh + + +# setup letsencrypt certificate +RUN apt-get install -y certbot python3-certbot-nginx +COPY hosting/letsencrypt /app/letsencrypt +RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh + +COPY --from=build /app/node_modules /node_modules +COPY --from=build /app/package.json /package.json +COPY --from=build /app/packages/server /app +COPY --from=build /app/packages/worker /worker +COPY --from=build /app/packages/string-templates /string-templates + +RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates + + +EXPOSE 80 +EXPOSE 443 +# Expose port 2222 for SSH on Azure App Service build +EXPOSE 2222 +VOLUME /data + + +HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh" + +# must set this just before running +ENV NODE_ENV=production +WORKDIR / + +CMD ["./runner.sh"] diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index d980202f88..9dc7aa25d8 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -7,16 +7,16 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000 +[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app -[[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379 +[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1 [[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002 -[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://localhost:4002 -[[ -z "${APPS_URL}" ]] && export APPS_URL=http://localhost:4001 +[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002 +[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001 [[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app # export CUSTOM_DOMAIN=budi001.custom.com @@ -51,7 +51,7 @@ do fi done if [[ -z "${COUCH_DB_URL}" ]]; then - export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984 + export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984 fi if [ ! -f "${DATA_DIR}/.env" ]; then touch ${DATA_DIR}/.env diff --git a/lerna.json b/lerna.json index 0dd6abdc46..62a762ec71 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.34", + "version": "2.11.35", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/nx.json b/nx.json index 8176bae82c..fef6893f9d 100644 --- a/nx.json +++ b/nx.json @@ -8,5 +8,9 @@ } } }, - "targetDefaults": {} + "targetDefaults": { + "build": { + "inputs": ["{workspaceRoot}/scripts/build.js"] + } + } } diff --git a/package.json b/package.json index c38ef76e17..7f5c971009 100644 --- a/package.json +++ b/package.json @@ -3,14 +3,11 @@ "private": true, "devDependencies": { "@esbuild-plugins/tsconfig-paths": "^0.1.2", - "@nx/js": "16.4.3", - "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "6.7.2", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", "eslint": "^8.44.0", "husky": "^8.0.3", - "js-yaml": "^4.1.0", "kill-port": "^1.6.1", "lerna": "7.1.1", "madge": "^6.0.0", @@ -19,8 +16,6 @@ "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", - "rimraf": "^3.0.2", - "rollup-plugin-replace": "^2.2.0", "svelte": "3.49.0", "typescript": "5.2.2", "@babel/core": "^7.22.5", @@ -51,7 +46,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages qa-core --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -61,7 +56,6 @@ "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:pre": "yarn build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", @@ -69,8 +63,7 @@ "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image", + "build:docker:single": "./scripts/build-single-image.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index e9eb77df7c..22ca5b21cc 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -26,7 +26,7 @@ "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", "@techpass/passport-openidconnect": "0.3.2", - "aws-cloudfront-sign": "2.2.0", + "aws-cloudfront-sign": "3.0.2", "aws-sdk": "2.1030.0", "bcrypt": "5.1.0", "bcryptjs": "2.4.3", diff --git a/packages/backend-core/src/objectStore/cloudfront.ts b/packages/backend-core/src/objectStore/cloudfront.ts index a61ea7f583..866fe9e880 100644 --- a/packages/backend-core/src/objectStore/cloudfront.ts +++ b/packages/backend-core/src/objectStore/cloudfront.ts @@ -1,5 +1,5 @@ import env from "../environment" -const cfsign = require("aws-cloudfront-sign") +import * as cfsign from "aws-cloudfront-sign" let PRIVATE_KEY: string | undefined @@ -21,7 +21,7 @@ function getPrivateKey() { const getCloudfrontSignParams = () => { return { - keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID, + keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!, privateKeyString: getPrivateKey(), expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour } diff --git a/packages/backend-core/src/plugin/utils.ts b/packages/backend-core/src/plugin/utils.ts index f73ded0659..8974a9f5a2 100644 --- a/packages/backend-core/src/plugin/utils.ts +++ b/packages/backend-core/src/plugin/utils.ts @@ -6,6 +6,7 @@ import { AutomationStepIdArray, AutomationIOType, AutomationCustomIOType, + DatasourceFeature, } from "@budibase/types" import joi from "joi" @@ -67,9 +68,27 @@ function validateDatasource(schema: any) { version: joi.string().optional(), schema: joi.object({ docs: joi.string(), + plus: joi.boolean().optional(), + isSQL: joi.boolean().optional(), + auth: joi + .object({ + type: joi.string().required(), + }) + .optional(), + features: joi + .object( + Object.fromEntries( + Object.values(DatasourceFeature).map(key => [ + key, + joi.boolean().optional(), + ]) + ) + ) + .optional(), + relationships: joi.boolean().optional(), + description: joi.string().required(), friendlyName: joi.string().required(), type: joi.string().allow(...DATASOURCE_TYPES), - description: joi.string().required(), datasource: joi.object().pattern(joi.string(), fieldValidator).required(), query: joi .object() diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 4791776c57..78eed2b608 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -82,9 +82,9 @@ "@spectrum-css/vars": "3.0.1", "dayjs": "^1.10.8", "easymde": "^2.16.1", + "svelte-dnd-action": "^0.9.8", "svelte-flatpickr": "3.2.3", - "svelte-portal": "^1.0.0", - "svelte-dnd-action": "^0.9.8" + "svelte-portal": "^1.0.0" }, "resolutions": { "loader-utils": "1.4.1" diff --git a/packages/builder/build/copy.js b/packages/builder/build/copy.js deleted file mode 100644 index f3077b7854..0000000000 --- a/packages/builder/build/copy.js +++ /dev/null @@ -1,8 +0,0 @@ -const ncp = require("ncp").ncp - -ncp("./dist", "../server/builder", function (err) { - if (err) { - return console.error(err) - } - console.log("Copied dist folder to ../server/builder") -}) diff --git a/packages/builder/package.json b/packages/builder/package.json index 20530cad20..23a1a1ecd2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -85,8 +85,8 @@ "@babel/core": "^7.12.14", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", - "@rollup/plugin-replace": "^2.4.2", - "@roxi/routify": "2.18.5", + "@rollup/plugin-replace": "^5.0.3", + "@roxi/routify": "2.18.12", "@sveltejs/vite-plugin-svelte": "1.0.1", "@testing-library/jest-dom": "5.17.0", "@testing-library/svelte": "^3.2.2", @@ -95,16 +95,18 @@ "jest": "29.6.2", "jsdom": "^21.1.1", "ncp": "^2.0.0", - "rollup": "^2.44.0", "svelte": "^3.48.0", "svelte-jester": "^1.3.2", - "vite": "^3.0.8", - "vite-plugin-static-copy": "^0.16.0", + "vite": "^4.4.11", + "vite-plugin-static-copy": "^0.17.0", "vitest": "^0.29.2" }, "nx": { "targets": { "build": { + "outputs": [ + "{workspaceRoot}/packages/server/builder" + ], "dependsOn": [ { "projects": [ diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ba61ede746..7b51e6c839 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -36,7 +36,7 @@ import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" - const AUTO_TYPE = "auto" + const AUTO_TYPE = FIELDS.AUTO.type const FORMULA_TYPE = FIELDS.FORMULA.type const LINK_TYPE = FIELDS.LINK.type const STRING_TYPE = FIELDS.STRING.type @@ -60,8 +60,13 @@ {} ) - function makeFieldId(type, subtype) { - return `${type}${subtype || ""}`.toUpperCase() + function makeFieldId(type, subtype, autocolumn) { + // don't make field IDs for auto types + if (type === AUTO_TYPE || autocolumn) { + return type.toUpperCase() + } else { + return `${type}${subtype || ""}`.toUpperCase() + } } let originalName @@ -183,7 +188,8 @@ if (!savingColumn) { editableColumn.fieldId = makeFieldId( editableColumn.type, - editableColumn.subtype + editableColumn.subtype, + editableColumn.autocolumn ) allowedTypes = getAllowedTypes().map(t => ({ @@ -419,7 +425,7 @@ FIELDS.FORMULA, FIELDS.JSON, isUsers ? FIELDS.USERS : FIELDS.USER, - { name: "Auto Column", type: AUTO_TYPE }, + FIELDS.AUTO, ] } else { let fields = [ @@ -538,7 +544,7 @@ getOptionValue={field => field.fieldId} getOptionIcon={field => field.icon} isOptionEnabled={option => { - if (option.type == AUTO_TYPE) { + if (option.type === AUTO_TYPE) { return availableAutoColumnKeys?.length > 0 } return true diff --git a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte index 9e42dfecd9..abec380b46 100644 --- a/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte +++ b/packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte @@ -13,6 +13,8 @@ import { Helpers } from "@budibase/bbui" import { RelationshipErrorChecker } from "./relationshipErrors" import { onMount } from "svelte" + import RelationshipSelector from "components/common/RelationshipSelector.svelte" + import { PrettyRelationshipDefinitions } from "constants/backend" export let save export let datasource @@ -22,16 +24,21 @@ export let selectedFromTable export let close - const relationshipTypes = [ - { - label: "One to Many", - value: RelationshipType.MANY_TO_ONE, + let relationshipMap = { + [RelationshipType.MANY_TO_MANY]: { + part1: PrettyRelationshipDefinitions.MANY, + part2: PrettyRelationshipDefinitions.MANY, }, - { - label: "Many to Many", - value: RelationshipType.MANY_TO_MANY, + [RelationshipType.MANY_TO_ONE]: { + part1: PrettyRelationshipDefinitions.ONE, + part2: PrettyRelationshipDefinitions.MANY, }, - ] + } + let relationshipOpts1 = Object.values(PrettyRelationshipDefinitions) + let relationshipOpts2 = Object.values(PrettyRelationshipDefinitions) + + let relationshipPart1 = PrettyRelationshipDefinitions.MANY + let relationshipPart2 = PrettyRelationshipDefinitions.ONE let originalFromColumnName = toRelationship.name, originalToColumnName = fromRelationship.name @@ -49,14 +56,32 @@ ) let errors = {} let fromPrimary, fromForeign, fromColumn, toColumn - let fromId, toId, throughId, throughToKey, throughFromKey + + let throughId, throughToKey, throughFromKey let isManyToMany, isManyToOne, relationshipType let hasValidated = false + $: fromId = null + $: toId = null + $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id, + name: table.name, + _id: table._id, })) + + $: { + // Determine the relationship type based on the selected values of both parts + relationshipType = Object.entries(relationshipMap).find( + ([_, parts]) => + parts.part1 === relationshipPart1 && parts.part2 === relationshipPart2 + )?.[0] + + changed(() => { + hasValidated = false + }) + } $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType) $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY @@ -338,33 +363,34 @@ onConfirm={saveRelationship} disabled={!valid} > - - changed(() => { - const table = plusTables.find(tbl => tbl._id === e.detail) - fromColumn = table?.name || "" - fromPrimary = table?.primary?.[0] - })} - /> - {/if} + + + changed(() => { + const table = plusTables.find(tbl => tbl._id === e.detail) + fromColumn = table?.name || "" + fromPrimary = table?.primary?.[0] + })} + secondaryTableChanged={e => + changed(() => { + const table = plusTables.find(tbl => tbl._id === e.detail) + toColumn = table.name || "" + fromForeign = null + })} + /> + {#if isManyToOne && fromId} - changed(() => { - const table = plusTables.find(tbl => tbl._id === e.detail) - toColumn = table.name || "" - fromForeign = null - })} - /> {#if isManyToMany} table.name} getOptionValue={table => table._id} bind:value={relationshipTableIdPrimary} + on:change={primaryTableChanged} + bind:error={errors.fromTable} /> @@ -46,20 +52,24 @@ +{#if editableColumn} + +{/if}