MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into cheeks-lab-day-fields

This commit is contained in:
Andrew Kingston 2024-11-26 11:13:37 +00:00 committed by GitHub
parent 8cb0817ea3 aa133e9253
commit e382512c3a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
152 changed files with 21653 additions and 18515 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintignore
View File

@ -9,8 +9,5 @@ packages/server/client
packages/server/coverage packages/server/coverage
packages/builder/.routify packages/builder/.routify
packages/sdk/sdk packages/sdk/sdk
packages/account-portal/packages/server/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/ui/build
**/*.ivm.bundle.js **/*.ivm.bundle.js
packages/server/build/oldClientVersions/**/** packages/server/build/oldClientVersions/**/**

129
.github/workflows/budibase_ci.yml vendored
View File

@ -64,18 +64,15 @@ jobs:
- run: yarn --frozen-lockfile - run: yarn --frozen-lockfile
# Run build all the projects # Run build all the projects
- name: Build OSS - name: Build
run: yarn build:oss run: yarn build
- name: Build account portal
run: yarn build:account-portal
if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
# Check the types of the projects built via esbuild # Check the types of the projects built via esbuild
- name: Check types - name: Check types
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server yarn check:types --since=${{ env.NX_BASE_BRANCH }}
else else
yarn check:types --ignore @budibase/account-portal-server yarn check:types
fi fi
helm-lint: helm-lint:
@ -117,9 +114,11 @@ jobs:
- name: Test - name: Test
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
else else
yarn test --ignore=@budibase/worker --ignore=@budibase/server yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --no-prefix
fi fi
test-worker: test-worker:
@ -141,13 +140,22 @@ jobs:
- name: Test worker - name: Test worker
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }} AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
else if [ -z "$AFFECTED" ]; then
yarn test --scope=@budibase/worker echo "No affected tests to run"
exit 0
fi
fi fi
cd packages/worker
yarn test --verbose --reporters=default --reporters=github-actions
test-server: test-server:
runs-on: budi-tubby-tornado-quad-core-300gb runs-on: ubuntu-latest
strategy:
matrix:
datasource:
[mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
steps: steps:
- name: Checkout repo - name: Checkout repo
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -170,12 +178,19 @@ jobs:
- name: Pull testcontainers images - name: Pull testcontainers images
run: | run: |
docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }} & if [ "${{ matrix.datasource }}" == "mssql" ]; then
docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }} & docker pull mcr.microsoft.com/mssql/server@${{ steps.dotenv.outputs.MSSQL_SHA }}
docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }} & elif [ "${{ matrix.datasource }}" == "mysql" ]; then
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }} & docker pull mysql@${{ steps.dotenv.outputs.MYSQL_SHA }}
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }} & elif [ "${{ matrix.datasource }}" == "postgres" ]; then
docker pull budibase/oracle-database:23.2-slim-faststart & docker pull postgres@${{ steps.dotenv.outputs.POSTGRES_SHA }}
elif [ "${{ matrix.datasource }}" == "mongodb" ]; then
docker pull mongo@${{ steps.dotenv.outputs.MONGODB_SHA }}
elif [ "${{ matrix.datasource }}" == "mariadb" ]; then
docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
elif [ "${{ matrix.datasource }}" == "oracle" ]; then
docker pull budibase/oracle-database:23.2-slim-faststart
fi
docker pull minio/minio & docker pull minio/minio &
docker pull redis & docker pull redis &
docker pull testcontainers/ryuk:0.5.1 & docker pull testcontainers/ryuk:0.5.1 &
@ -186,13 +201,25 @@ jobs:
- run: yarn --frozen-lockfile - run: yarn --frozen-lockfile
- name: Test server - name: Test server
env:
DATASOURCE: ${{ matrix.datasource }}
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/server)
else if [ -z "$AFFECTED" ]; then
yarn test --scope=@budibase/server echo "No affected tests to run"
exit 0
fi
fi fi
FILTER="./src/tests/filters/datasource-tests.js"
if [ "${{ matrix.datasource }}" == "none" ]; then
FILTER="./src/tests/filters/non-datasource-tests.js"
fi
cd packages/server
yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
check-pro-submodule: check-pro-submodule:
runs-on: ubuntu-latest runs-on: ubuntu-latest
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
@ -252,64 +279,6 @@ jobs:
echo 'All good, the submodule had been merged and setup correctly!' echo 'All good, the submodule had been merged and setup correctly!'
fi fi
check-accountportal-submodule:
runs-on: ubuntu-latest
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v4
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
fetch-depth: 0
- uses: dorny/paths-filter@v3
id: changes
with:
filters: |
src:
- packages/account-portal/**
- if: steps.changes.outputs.src == 'true'
name: Check account portal commit
id: get_accountportal_commits
run: |
cd packages/account-portal
accountportal_commit=$(git rev-parse HEAD)
branch="${{ github.base_ref || github.ref_name }}"
echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
base_commit=$(git rev-parse origin/master)
if [[ ! -z $base_commit ]]; then
echo "target_branch=$branch"
echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
echo "accountportal_commit=$accountportal_commit"
echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
echo "base_commit=$base_commit"
echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
else
echo "Nothing to do - branch to branch merge."
fi
- name: Check submodule merged to base branch
if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';
if (submoduleCommit !== baseCommit) {
console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
process.exit(1);
} else {
console.log('All good, the submodule had been merged and setup correctly!')
}
check-lockfile: check-lockfile:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

3
.gitignore vendored
View File

@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/*
packages/builder/src/components/deploy/clientVersions.json packages/builder/src/components/deploy/clientVersions.json
packages/server/src/integrations/tests/utils/*.lock packages/server/src/integrations/tests/utils/*.lock
packages/builder/vite.config.mjs.timestamp* packages/builder/vite.config.mjs.timestamp*
packages/account-portal
# Logs # Logs
logs logs
@ -110,4 +111,4 @@ budibase-component
budibase-datasource budibase-datasource
*.iml *.iml
.nx .nx

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "packages/pro"] [submodule "packages/pro"]
path = packages/pro path = packages/pro
url = git@github.com:Budibase/budibase-pro.git url = git@github.com:Budibase/budibase-pro.git
[submodule "packages/account-portal"]
path = packages/account-portal
url = git@github.com:Budibase/account-portal.git

4
.prettierignore
View File

@ -9,8 +9,4 @@ packages/backend-core/coverage
packages/builder/.routify packages/builder/.routify
packages/sdk/sdk packages/sdk/sdk
packages/pro/coverage packages/pro/coverage
packages/account-portal/packages/ui/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/server/build
packages/account-portal/packages/server/coverage
**/*.ivm.bundle.js **/*.ivm.bundle.js

10
.vscode/launch.json vendored
View File

@ -20,16 +20,6 @@
"args": ["${workspaceFolder}/packages/worker/src/index.ts"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
"cwd": "${workspaceFolder}/packages/worker" "cwd": "${workspaceFolder}/packages/worker"
}, },
{
"name": "Camunda Worker",
"type": "node",
"request": "launch",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": [
"${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts"
],
"cwd": "${workspaceFolder}/packages/account-portal/packages/server"
},
{ {
"type": "chrome", "type": "chrome",
"request": "launch", "request": "launch",

6
examples/nextjs-api-sales/yarn.lock
View File

@ -423,9 +423,9 @@ core-js-pure@^3.20.2:
integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ== integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ==
cross-spawn@^7.0.2: cross-spawn@^7.0.2:
version "7.0.3" version "7.0.6"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
dependencies: dependencies:
path-key "^3.1.0" path-key "^3.1.0"
shebang-command "^2.0.0" shebang-command "^2.0.0"

2
globalSetup.ts
View File

@ -62,6 +62,7 @@ export default async function setup() {
}, },
]) ])
.withLabels({ "com.budibase": "true" }) .withLabels({ "com.budibase": "true" })
.withTmpFs({ "/data": "rw" })
.withReuse() .withReuse()
.withWaitStrategy( .withWaitStrategy(
Wait.forSuccessfulCommand( Wait.forSuccessfulCommand(
@ -72,6 +73,7 @@ export default async function setup() {
const minio = new GenericContainer("minio/minio") const minio = new GenericContainer("minio/minio")
.withExposedPorts(9000) .withExposedPorts(9000)
.withCommand(["server", "/data"]) .withCommand(["server", "/data"])
.withTmpFs({ "/data": "rw" })
.withEnvironment({ .withEnvironment({
MINIO_ACCESS_KEY: "budibase", MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase", MINIO_SECRET_KEY: "budibase",

14
hosting/nginx.dev.conf
View File

@ -45,20 +45,6 @@ http {
client_max_body_size 50000m; client_max_body_size 50000m;
ignore_invalid_headers off; ignore_invalid_headers off;
proxy_buffering off; proxy_buffering off;
set $csp_default "default-src 'self'";
set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
set $csp_manifest "manifest-src 'self'";
set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live";
set $csp_worker "worker-src blob:";
add_header Content-Security-Policy "${csp_default}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
error_page 502 503 504 /error.html; error_page 502 503 504 /error.html;
location = /error.html { location = /error.html {

7
lerna.json
View File

@ -1,12 +1,7 @@
{ {
"$schema": "node_modules/lerna/schemas/lerna-schema.json", "$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.3", "version": "3.2.12",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [
"packages/*",
"!packages/account-portal",
"packages/account-portal/packages/*"
],
"concurrency": 20, "concurrency": 20,
"command": { "command": {
"publish": { "publish": {

1
nx.json
View File

@ -2,7 +2,6 @@
"$schema": "./node_modules/nx/schemas/nx-schema.json", "$schema": "./node_modules/nx/schemas/nx-schema.json",
"tasksRunnerOptions": { "tasksRunnerOptions": {
"default": { "default": {
"runner": "nx-cloud",
"options": { "options": {
"cacheableOperations": ["build", "test", "check:types"] "cacheableOperations": ["build", "test", "check:types"]
} }

29
package.json
View File

@ -9,6 +9,7 @@
"@types/node": "20.10.0", "@types/node": "20.10.0",
"@types/proper-lockfile": "^4.1.4", "@types/proper-lockfile": "^4.1.4",
"@typescript-eslint/parser": "6.9.0", "@typescript-eslint/parser": "6.9.0",
"depcheck": "^1.4.7",
"esbuild": "^0.18.17", "esbuild": "^0.18.17",
"esbuild-node-externals": "^1.14.0", "esbuild-node-externals": "^1.14.0",
"eslint": "^8.52.0", "eslint": "^8.52.0",
@ -24,22 +25,22 @@
"prettier": "2.8.8", "prettier": "2.8.8",
"prettier-plugin-svelte": "^2.3.0", "prettier-plugin-svelte": "^2.3.0",
"proper-lockfile": "^4.1.2", "proper-lockfile": "^4.1.2",
"svelte": "^4.2.10", "svelte": "4.2.19",
"svelte-eslint-parser": "^0.33.1", "svelte-eslint-parser": "^0.33.1",
"typescript": "5.5.2", "typescript": "5.5.2",
"typescript-eslint": "^7.3.1", "typescript-eslint": "^7.3.1",
"yargs": "^17.7.2" "yargs": "^17.7.2",
"cross-spawn": "7.0.6"
}, },
"scripts": { "scripts": {
"get-past-client-version": "node scripts/getPastClientVersion.js", "get-past-client-version": "node scripts/getPastClientVersion.js",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
"build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
"build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker", "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
"build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:cli": "yarn build --scope @budibase/cli", "build:cli": "yarn build --scope @budibase/cli",
"build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
"build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
"check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server", "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types",
"check:dependencies": "lerna run --concurrency 2 check:dependencies",
"build:sdk": "lerna run --stream build:sdk", "build:sdk": "lerna run --stream build:sdk",
"deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
"release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset", "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
@ -52,15 +53,12 @@
"kill-server": "kill-port 4001 4002", "kill-server": "kill-port 4001 4002",
"kill-accountportal": "kill-port 3001 4003", "kill-accountportal": "kill-port 3001 4003",
"kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal", "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
"dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev",
"dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
"dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server", "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
"dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
"dev:camunda": "./scripts/deploy-camunda.sh",
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "./scripts/devDocker.sh", "dev:docker": "./scripts/devDocker.sh",
"test": "lerna run --concurrency 1 --stream test --stream", "test": "lerna run --concurrency 1 --stream test",
"test:containers:kill": "./scripts/killTestcontainers.sh", "test:containers:kill": "./scripts/killTestcontainers.sh",
"lint:eslint": "eslint packages --max-warnings=0", "lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
@ -98,9 +96,7 @@
}, },
"workspaces": { "workspaces": {
"packages": [ "packages": [
"packages/*", "packages/*"
"!packages/account-portal",
"packages/account-portal/packages/*"
] ]
}, },
"resolutions": { "resolutions": {
@ -114,7 +110,7 @@
"semver": "7.5.3", "semver": "7.5.3",
"http-cache-semantics": "4.1.1", "http-cache-semantics": "4.1.1",
"msgpackr": "1.10.1", "msgpackr": "1.10.1",
"axios": "1.6.3", "axios": "1.7.7",
"xml2js": "0.6.2", "xml2js": "0.6.2",
"unset-value": "2.0.1", "unset-value": "2.0.1",
"passport": "0.6.0", "passport": "0.6.0",
@ -124,6 +120,5 @@
}, },
"engines": { "engines": {
"node": ">=20.0.0 <21.0.0" "node": ">=20.0.0 <21.0.0"
}, }
"dependencies": {}
} }

1
packages/account-portal

@ -1 +0,0 @@
Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140

4
packages/backend-core/.npmignore
View File

@ -1,6 +1,4 @@
* *
!dist/**/* !dist/**/*
dist/tsconfig.build.tsbuildinfo dist/tsconfig.build.tsbuildinfo
!package.json !package.json
!src/**
!tests/**

28
packages/backend-core/package.json
View File

@ -9,6 +9,13 @@
"./tests": "./dist/tests/index.js", "./tests": "./dist/tests/index.js",
"./*": "./dist/*.js" "./*": "./dist/*.js"
}, },
"typesVersions": {
"*": {
"tests": [
"dist/tests/index.d.ts"
]
}
},
"author": "Budibase", "author": "Budibase",
"license": "GPL-3.0", "license": "GPL-3.0",
"scripts": { "scripts": {
@ -17,6 +24,7 @@
"build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js", "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
"check:dependencies": "node ../../scripts/depcheck.js",
"test": "bash scripts/test.sh", "test": "bash scripts/test.sh",
"test:watch": "jest --watchAll" "test:watch": "jest --watchAll"
}, },
@ -25,17 +33,21 @@
"@budibase/pouchdb-replication-stream": "1.2.11", "@budibase/pouchdb-replication-stream": "1.2.11",
"@budibase/shared-core": "0.0.0", "@budibase/shared-core": "0.0.0",
"@budibase/types": "0.0.0", "@budibase/types": "0.0.0",
"@techpass/passport-openidconnect": "0.3.3",
"aws-cloudfront-sign": "3.0.2", "aws-cloudfront-sign": "3.0.2",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"correlation-id": "4.0.0", "correlation-id": "4.0.0",
"dd-trace": "5.2.0", "dd-trace": "5.26.0",
"dotenv": "16.0.1", "dotenv": "16.0.1",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"joi": "17.6.0", "joi": "17.6.0",
"jsonwebtoken": "9.0.2", "jsonwebtoken": "9.0.2",
"knex": "2.4.2",
"koa-passport": "^6.0.0", "koa-passport": "^6.0.0",
"koa-pino-logger": "4.0.0", "koa-pino-logger": "4.0.0",
"lodash": "4.17.21", "lodash": "4.17.21",
@ -46,17 +58,17 @@
"pino": "8.11.0", "pino": "8.11.0",
"pino-http": "8.3.3", "pino-http": "8.3.3",
"posthog-node": "4.0.1", "posthog-node": "4.0.1",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redlock": "4.2.0", "redlock": "4.2.0",
"rotating-file-stream": "3.1.0", "rotating-file-stream": "3.1.0",
"sanitize-s3-objectkey": "0.0.1", "sanitize-s3-objectkey": "0.0.1",
"semver": "^7.5.4", "semver": "^7.5.4",
"tar-fs": "2.1.1", "tar-fs": "2.1.1",
"uuid": "^8.3.2", "uuid": "^8.3.2"
"knex": "2.4.2"
}, },
"devDependencies": { "devDependencies": {
"@jest/types": "^29.6.3",
"@shopify/jest-koa-mocks": "5.1.1", "@shopify/jest-koa-mocks": "5.1.1",
"@swc/core": "1.3.71", "@swc/core": "1.3.71",
"@swc/jest": "0.2.27", "@swc/jest": "0.2.27",
@ -64,8 +76,9 @@
"@types/cookies": "0.7.8", "@types/cookies": "0.7.8",
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/pouchdb": "6.4.0", "@types/pouchdb": "6.4.2",
"@types/redlock": "4.0.7", "@types/redlock": "4.0.7",
"@types/semver": "7.3.7", "@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1", "@types/tar-fs": "2.0.1",
@ -74,6 +87,7 @@
"ioredis-mock": "8.9.0", "ioredis-mock": "8.9.0",
"jest": "29.7.0", "jest": "29.7.0",
"jest-serial-runner": "1.2.1", "jest-serial-runner": "1.2.1",
"nock": "^13.5.6",
"pino-pretty": "10.0.0", "pino-pretty": "10.0.0",
"pouchdb-adapter-memory": "7.2.2", "pouchdb-adapter-memory": "7.2.2",
"testcontainers": "^10.7.2", "testcontainers": "^10.7.2",

11
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -10,7 +10,6 @@ import {
DatabaseQueryOpts, DatabaseQueryOpts,
DBError, DBError,
Document, Document,
FeatureFlag,
isDocument, isDocument,
RowResponse, RowResponse,
RowValue, RowValue,
@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
import { DDInstrumentedDatabase } from "../instrumentation" import { DDInstrumentedDatabase } from "../instrumentation"
import { checkSlashesInUrl } from "../../helpers" import { checkSlashesInUrl } from "../../helpers"
import { sqlLog } from "../../sql/utils" import { sqlLog } from "../../sql/utils"
import { flags } from "../../features"
const DATABASE_NOT_FOUND = "Database does not exist." const DATABASE_NOT_FOUND = "Database does not exist."
@ -192,7 +190,7 @@ export class DatabaseImpl implements Database {
} }
} }
private async performCall<T>(call: DBCallback<T>): Promise<any> { private async performCall<T>(call: DBCallback<T>): Promise<T> {
const db = this.getDb() const db = this.getDb()
const fnc = await call(db) const fnc = await call(db)
try { try {
@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
} }
async destroy() { async destroy() {
if ( if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
(await flags.isEnabled(FeatureFlag.SQS)) &&
(await this.exists(SQLITE_DESIGN_DOC_ID))
) {
// delete the design document, then run the cleanup operation // delete the design document, then run the cleanup operation
const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID) const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
// remove all tables - save the definition then trigger a cleanup // remove all tables - save the definition then trigger a cleanup
@ -472,7 +467,7 @@ export class DatabaseImpl implements Database {
} catch (err: any) { } catch (err: any) {
// didn't exist, don't worry // didn't exist, don't worry
if (err.statusCode === 404) { if (err.statusCode === 404) {
return return { ok: true }
} else { } else {
throw new CouchDBError(err.message, err) throw new CouchDBError(err.message, err)
} }

125
packages/backend-core/src/db/instrumentation.ts
View File

@ -27,7 +27,7 @@ export class DDInstrumentedDatabase implements Database {
exists(docId?: string): Promise<boolean> { exists(docId?: string): Promise<boolean> {
return tracer.trace("db.exists", span => { return tracer.trace("db.exists", span => {
span?.addTags({ db_name: this.name, doc_id: docId }) span.addTags({ db_name: this.name, doc_id: docId })
if (docId) { if (docId) {
return this.db.exists(docId) return this.db.exists(docId)
} }
@ -37,15 +37,17 @@ export class DDInstrumentedDatabase implements Database {
get<T extends Document>(id?: string | undefined): Promise<T> { get<T extends Document>(id?: string | undefined): Promise<T> {
return tracer.trace("db.get", span => { return tracer.trace("db.get", span => {
span?.addTags({ db_name: this.name, doc_id: id }) span.addTags({ db_name: this.name, doc_id: id })
return this.db.get(id) return this.db.get(id)
}) })
} }
tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> { tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
return tracer.trace("db.tryGet", span => { return tracer.trace("db.tryGet", async span => {
span?.addTags({ db_name: this.name, doc_id: id }) span.addTags({ db_name: this.name, doc_id: id })
return this.db.tryGet(id) const doc = await this.db.tryGet<T>(id)
span.addTags({ doc_found: doc !== undefined })
return doc
}) })
} }
@ -53,13 +55,15 @@ export class DDInstrumentedDatabase implements Database {
ids: string[], ids: string[],
opts?: { allowMissing?: boolean | undefined } | undefined opts?: { allowMissing?: boolean | undefined } | undefined
): Promise<T[]> { ): Promise<T[]> {
return tracer.trace("db.getMultiple", span => { return tracer.trace("db.getMultiple", async span => {
span?.addTags({ span.addTags({
db_name: this.name, db_name: this.name,
num_docs: ids.length, num_docs: ids.length,
allow_missing: opts?.allowMissing, allow_missing: opts?.allowMissing,
}) })
return this.db.getMultiple(ids, opts) const docs = await this.db.getMultiple<T>(ids, opts)
span.addTags({ num_docs_found: docs.length })
return docs
}) })
} }
@ -69,12 +73,14 @@ export class DDInstrumentedDatabase implements Database {
idOrDoc: string | Document, idOrDoc: string | Document,
rev?: string rev?: string
): Promise<DocumentDestroyResponse> { ): Promise<DocumentDestroyResponse> {
return tracer.trace("db.remove", span => { return tracer.trace("db.remove", async span => {
span?.addTags({ db_name: this.name, doc_id: idOrDoc }) span.addTags({ db_name: this.name, doc_id: idOrDoc, rev })
const isDocument = typeof idOrDoc === "object" const isDocument = typeof idOrDoc === "object"
const id = isDocument ? idOrDoc._id! : idOrDoc const id = isDocument ? idOrDoc._id! : idOrDoc
rev = isDocument ? idOrDoc._rev : rev rev = isDocument ? idOrDoc._rev : rev
return this.db.remove(id, rev) const resp = await this.db.remove(id, rev)
span.addTags({ ok: resp.ok })
return resp
}) })
} }
@ -83,7 +89,11 @@ export class DDInstrumentedDatabase implements Database {
opts?: { silenceErrors?: boolean } opts?: { silenceErrors?: boolean }
): Promise<void> { ): Promise<void> {
return tracer.trace("db.bulkRemove", span => { return tracer.trace("db.bulkRemove", span => {
span?.addTags({ db_name: this.name, num_docs: documents.length }) span.addTags({
db_name: this.name,
num_docs: documents.length,
silence_errors: opts?.silenceErrors,
})
return this.db.bulkRemove(documents, opts) return this.db.bulkRemove(documents, opts)
}) })
} }
@ -92,15 +102,21 @@ export class DDInstrumentedDatabase implements Database {
document: AnyDocument, document: AnyDocument,
opts?: DatabasePutOpts | undefined opts?: DatabasePutOpts | undefined
): Promise<DocumentInsertResponse> { ): Promise<DocumentInsertResponse> {
return tracer.trace("db.put", span => { return tracer.trace("db.put", async span => {
span?.addTags({ db_name: this.name, doc_id: document._id }) span.addTags({
return this.db.put(document, opts) db_name: this.name,
doc_id: document._id,
force: opts?.force,
})
const resp = await this.db.put(document, opts)
span.addTags({ ok: resp.ok })
return resp
}) })
} }
bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> { bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
return tracer.trace("db.bulkDocs", span => { return tracer.trace("db.bulkDocs", span => {
span?.addTags({ db_name: this.name, num_docs: documents.length }) span.addTags({ db_name: this.name, num_docs: documents.length })
return this.db.bulkDocs(documents) return this.db.bulkDocs(documents)
}) })
} }
@ -108,9 +124,15 @@ export class DDInstrumentedDatabase implements Database {
allDocs<T extends Document | RowValue>( allDocs<T extends Document | RowValue>(
params: DatabaseQueryOpts params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> { ): Promise<AllDocsResponse<T>> {
return tracer.trace("db.allDocs", span => { return tracer.trace("db.allDocs", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, ...params })
return this.db.allDocs(params) const resp = await this.db.allDocs<T>(params)
span.addTags({
total_rows: resp.total_rows,
rows_length: resp.rows.length,
offset: resp.offset,
})
return resp
}) })
} }
@ -118,57 +140,75 @@ export class DDInstrumentedDatabase implements Database {
viewName: string, viewName: string,
params: DatabaseQueryOpts params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> { ): Promise<AllDocsResponse<T>> {
return tracer.trace("db.query", span => { return tracer.trace("db.query", async span => {
span?.addTags({ db_name: this.name, view_name: viewName }) span.addTags({ db_name: this.name, view_name: viewName, ...params })
return this.db.query(viewName, params) const resp = await this.db.query<T>(viewName, params)
span.addTags({
total_rows: resp.total_rows,
rows_length: resp.rows.length,
offset: resp.offset,
})
return resp
}) })
} }
destroy(): Promise<void | OkResponse> { destroy(): Promise<OkResponse> {
return tracer.trace("db.destroy", span => { return tracer.trace("db.destroy", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.destroy() const resp = await this.db.destroy()
span.addTags({ ok: resp.ok })
return resp
}) })
} }
compact(): Promise<void | OkResponse> { compact(): Promise<OkResponse> {
return tracer.trace("db.compact", span => { return tracer.trace("db.compact", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.compact() const resp = await this.db.compact()
span.addTags({ ok: resp.ok })
return resp
}) })
} }
dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> { dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
return tracer.trace("db.dump", span => { return tracer.trace("db.dump", span => {
span?.addTags({ db_name: this.name }) span.addTags({
db_name: this.name,
batch_limit: opts?.batch_limit,
batch_size: opts?.batch_size,
style: opts?.style,
timeout: opts?.timeout,
num_doc_ids: opts?.doc_ids?.length,
view: opts?.view,
})
return this.db.dump(stream, opts) return this.db.dump(stream, opts)
}) })
} }
load(...args: any[]): Promise<any> { load(...args: any[]): Promise<any> {
return tracer.trace("db.load", span => { return tracer.trace("db.load", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.load(...args) return this.db.load(...args)
}) })
} }
createIndex(...args: any[]): Promise<any> { createIndex(...args: any[]): Promise<any> {
return tracer.trace("db.createIndex", span => { return tracer.trace("db.createIndex", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.createIndex(...args) return this.db.createIndex(...args)
}) })
} }
deleteIndex(...args: any[]): Promise<any> { deleteIndex(...args: any[]): Promise<any> {
return tracer.trace("db.deleteIndex", span => { return tracer.trace("db.deleteIndex", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.deleteIndex(...args) return this.db.deleteIndex(...args)
}) })
} }
getIndexes(...args: any[]): Promise<any> { getIndexes(...args: any[]): Promise<any> {
return tracer.trace("db.getIndexes", span => { return tracer.trace("db.getIndexes", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.getIndexes(...args) return this.db.getIndexes(...args)
}) })
} }
@ -177,22 +217,27 @@ export class DDInstrumentedDatabase implements Database {
sql: string, sql: string,
parameters?: SqlQueryBinding parameters?: SqlQueryBinding
): Promise<T[]> { ): Promise<T[]> {
return tracer.trace("db.sql", span => { return tracer.trace("db.sql", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_bindings: parameters?.length })
return this.db.sql(sql, parameters) const resp = await this.db.sql<T>(sql, parameters)
span.addTags({ num_rows: resp.length })
return resp
}) })
} }
sqlPurgeDocument(docIds: string[] | string): Promise<void> { sqlPurgeDocument(docIds: string[] | string): Promise<void> {
return tracer.trace("db.sqlPurgeDocument", span => { return tracer.trace("db.sqlPurgeDocument", span => {
span?.addTags({ db_name: this.name }) span.addTags({
db_name: this.name,
num_docs: Array.isArray(docIds) ? docIds.length : 1,
})
return this.db.sqlPurgeDocument(docIds) return this.db.sqlPurgeDocument(docIds)
}) })
} }
sqlDiskCleanup(): Promise<void> { sqlDiskCleanup(): Promise<void> {
return tracer.trace("db.sqlDiskCleanup", span => { return tracer.trace("db.sqlDiskCleanup", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.sqlDiskCleanup() return this.db.sqlDiskCleanup()
}) })
} }

19
packages/backend-core/src/environment.ts
View File

@ -1,6 +1,7 @@
import { existsSync, readFileSync } from "fs" import { existsSync, readFileSync } from "fs"
import { ServiceType } from "@budibase/types" import { ServiceType } from "@budibase/types"
import { cloneDeep } from "lodash" import { cloneDeep } from "lodash"
import { createSecretKey } from "crypto"
function isTest() { function isTest() {
return isJest() return isJest()
@ -18,6 +19,12 @@ function isDev() {
return process.env.NODE_ENV !== "production" return process.env.NODE_ENV !== "production"
} }
function parseIntSafe(number?: string) {
if (number) {
return parseInt(number)
}
}
let LOADED = false let LOADED = false
if (!LOADED && isDev() && !isTest()) { if (!LOADED && isDev() && !isTest()) {
require("dotenv").config() require("dotenv").config()
@ -126,8 +133,12 @@ const environment = {
}, },
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
JS_BCRYPT: process.env.JS_BCRYPT, JS_BCRYPT: process.env.JS_BCRYPT,
JWT_SECRET: process.env.JWT_SECRET, JWT_SECRET: process.env.JWT_SECRET
JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK, ? createSecretKey(Buffer.from(process.env.JWT_SECRET))
: undefined,
JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK
? createSecretKey(Buffer.from(process.env.JWT_SECRET_FALLBACK))
: undefined,
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(), API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
@ -226,9 +237,7 @@ const environment = {
MIN_VERSION_WITHOUT_POWER_ROLE: MIN_VERSION_WITHOUT_POWER_ROLE:
process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0", process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY, DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
// stopgap migration strategy until we can ensure backwards compat without unsafe-inline in CSP BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
DISABLE_CSP_UNSAFE_INLINE_SCRIPTS:
process.env.DISABLE_CSP_UNSAFE_INLINE_SCRIPTS,
} }
export function setEnv(newEnvVars: Partial<typeof environment>): () => void { export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

2
packages/backend-core/src/features/features.ts
View File

@ -269,8 +269,6 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({ export const flags = new FlagSet({
[FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true), [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
[FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true), [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
[FeatureFlag.SQS]: Flag.boolean(true),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
[FeatureFlag.BUDIBASE_AI]: Flag.boolean(true), [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
}) })

5
packages/backend-core/src/middleware/contentSecurityPolicy.ts
View File

@ -1,5 +1,4 @@
import crypto from "crypto" import crypto from "crypto"
import env from "../environment"
const CSP_DIRECTIVES = { const CSP_DIRECTIVES = {
"default-src": ["'self'"], "default-src": ["'self'"],
@ -97,10 +96,6 @@ export async function contentSecurityPolicy(ctx: any, next: any) {
`'nonce-${nonce}'`, `'nonce-${nonce}'`,
] ]
if (!env.DISABLE_CSP_UNSAFE_INLINE_SCRIPTS) {
directives["script-src"].push("'unsafe-inline'")
}
ctx.state.nonce = nonce ctx.state.nonce = nonce
const cspHeader = Object.entries(directives) const cspHeader = Object.entries(directives)

2
packages/backend-core/src/security/tests/encryption.spec.ts
View File

@ -4,7 +4,7 @@ import env from "../../environment"
describe("encryption", () => { describe("encryption", () => {
it("should throw an error if API encryption key is not set", () => { it("should throw an error if API encryption key is not set", () => {
const jwt = getSecret(SecretOption.API) const jwt = getSecret(SecretOption.API)
expect(jwt).toBe(env.JWT_SECRET) expect(jwt).toBe(env.JWT_SECRET?.export().toString())
}) })
it("should throw an error if encryption key is not set", () => { it("should throw an error if encryption key is not set", () => {

1
packages/bbui/package.json
View File

@ -81,6 +81,7 @@
"@spectrum-css/typography": "3.0.1", "@spectrum-css/typography": "3.0.1",
"@spectrum-css/underlay": "2.0.9", "@spectrum-css/underlay": "2.0.9",
"@spectrum-css/vars": "3.0.1", "@spectrum-css/vars": "3.0.1",
"atrament": "^4.3.0",
"dayjs": "^1.10.8", "dayjs": "^1.10.8",
"easymde": "^2.16.1", "easymde": "^2.16.1",
"svelte-dnd-action": "^0.9.8", "svelte-dnd-action": "^0.9.8",

39
packages/bbui/src/Form/Core/Dropzone.svelte
View File

@ -8,6 +8,7 @@
import Link from "../../Link/Link.svelte" import Link from "../../Link/Link.svelte"
import Tag from "../../Tags/Tag.svelte" import Tag from "../../Tags/Tag.svelte"
import Tags from "../../Tags/Tags.svelte" import Tags from "../../Tags/Tags.svelte"
import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
const BYTES_IN_KB = 1000 const BYTES_IN_KB = 1000
const BYTES_IN_MB = 1000000 const BYTES_IN_MB = 1000000
@ -39,12 +40,14 @@
"jfif", "jfif",
"webp", "webp",
] ]
const fieldId = id || uuid() const fieldId = id || uuid()
let selectedImageIdx = 0 let selectedImageIdx = 0
let fileDragged = false let fileDragged = false
let selectedUrl let selectedUrl
let fileInput let fileInput
let loading = false
$: selectedImage = value?.[selectedImageIdx] ?? null $: selectedImage = value?.[selectedImageIdx] ?? null
$: fileCount = value?.length ?? 0 $: fileCount = value?.length ?? 0
$: isImage = $: isImage =
@ -86,10 +89,15 @@
} }
if (processFiles) { if (processFiles) {
const processedFiles = await processFiles(fileList) loading = true
const newValue = [...value, ...processedFiles] try {
dispatch("change", newValue) const processedFiles = await processFiles(fileList)
selectedImageIdx = newValue.length - 1 const newValue = [...value, ...processedFiles]
dispatch("change", newValue)
selectedImageIdx = newValue.length - 1
} finally {
loading = false
}
} else { } else {
dispatch("change", fileList) dispatch("change", fileList)
} }
@ -227,7 +235,7 @@
{#if showDropzone} {#if showDropzone}
<div <div
class="spectrum-Dropzone" class="spectrum-Dropzone"
class:disabled class:disabled={disabled || loading}
role="region" role="region"
tabindex="0" tabindex="0"
on:dragover={handleDragOver} on:dragover={handleDragOver}
@ -241,7 +249,7 @@
id={fieldId} id={fieldId}
{disabled} {disabled}
type="file" type="file"
multiple multiple={maximum !== 1}
accept={extensions} accept={extensions}
bind:this={fileInput} bind:this={fileInput}
on:change={handleFile} on:change={handleFile}
@ -339,6 +347,12 @@
{/if} {/if}
{/if} {/if}
</div> </div>
{#if loading}
<div class="loading">
<ProgressCircle size="M" />
</div>
{/if}
</div> </div>
{/if} {/if}
</div> </div>
@ -464,6 +478,7 @@
.spectrum-Dropzone { .spectrum-Dropzone {
height: 220px; height: 220px;
position: relative;
} }
.compact .spectrum-Dropzone { .compact .spectrum-Dropzone {
height: 40px; height: 40px;
@ -488,4 +503,14 @@
.tag { .tag {
margin-top: 8px; margin-top: 8px;
} }
.loading {
position: absolute;
display: grid;
place-items: center;
height: 100%;
width: 100%;
top: 0;
left: 0;
}
</style> </style>

16
packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte
View File

@ -1,4 +1,5 @@
<script> <script>
import { tick } from "svelte"
import { import {
ModalContent, ModalContent,
TextArea, TextArea,
@ -8,7 +9,6 @@
import { automationStore, selectedAutomation } from "stores/builder" import { automationStore, selectedAutomation } from "stores/builder"
import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { memo } from "@budibase/frontend-core"
import { AutomationEventType } from "@budibase/types" import { AutomationEventType } from "@budibase/types"
let failedParse = null let failedParse = null
@ -63,8 +63,7 @@
return true return true
} }
const memoTestData = memo(parseTestData($selectedAutomation.data.testData)) $: testData = testData || parseTestData($selectedAutomation.data.testData)
$: memoTestData.set(parseTestData($selectedAutomation.data.testData))
$: { $: {
// clone the trigger so we're not mutating the reference // clone the trigger so we're not mutating the reference
@ -83,7 +82,7 @@
$: isError = $: isError =
!isTriggerValid(trigger) || !isTriggerValid(trigger) ||
!(trigger.schema.outputs.required || []).every( !(trigger.schema.outputs.required || []).every(
required => $memoTestData?.[required] || required !== "row" required => testData?.[required] || required !== "row"
) )
function parseTestJSON(e) { function parseTestJSON(e) {
@ -110,11 +109,10 @@
} }
const testAutomation = async () => { const testAutomation = async () => {
// Ensure testData reactiveness is processed
await tick()
try { try {
await automationStore.actions.test( await automationStore.actions.test($selectedAutomation.data, testData)
$selectedAutomation.data,
$memoTestData
)
$automationStore.showTestPanel = true $automationStore.showTestPanel = true
} catch (error) { } catch (error) {
notifications.error(error) notifications.error(error)
@ -152,7 +150,7 @@
{#if selectedValues} {#if selectedValues}
<div class="tab-content-padding"> <div class="tab-content-padding">
<AutomationBlockSetup <AutomationBlockSetup
testData={$memoTestData} bind:testData
{schemaProperties} {schemaProperties}
isTestModal isTestModal
block={trigger} block={trigger}

10
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -503,7 +503,15 @@
row: { "Active": true, "Order Id" : 14, ... } row: { "Active": true, "Order Id" : 14, ... }
}) })
*/ */
const onChange = Utils.sequential(async update => { const onChange = async update => {
if (isTestModal) {
testData = update
}
updateAutomation(update)
}
const updateAutomation = Utils.sequential(async update => {
const request = cloneDeep(update) const request = cloneDeep(update)
// Process app trigger updates // Process app trigger updates
if (isTrigger && !isTestModal) { if (isTrigger && !isTestModal) {

7
packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
View File

@ -11,7 +11,6 @@
export let disabledPermissions = [] export let disabledPermissions = []
export let columns export let columns
export let fromRelationshipField export let fromRelationshipField
export let canSetRelationshipSchemas
const { datasource, dispatch } = getContext("grid") const { datasource, dispatch } = getContext("grid")
@ -129,6 +128,8 @@
} }
}) })
$: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK)
async function toggleColumn(column, permission) { async function toggleColumn(column, permission) {
const visible = permission !== FieldPermissions.HIDDEN const visible = permission !== FieldPermissions.HIDDEN
const readonly = permission === FieldPermissions.READONLY const readonly = permission === FieldPermissions.READONLY
@ -184,7 +185,7 @@
value={columnToPermissionOptions(column)} value={columnToPermissionOptions(column)}
options={column.options} options={column.options}
/> />
{#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN} {#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
<div class="relationship-columns"> <div class="relationship-columns">
<ActionButton <ActionButton
on:click={e => { on:click={e => {
@ -203,7 +204,7 @@
</div> </div>
</div> </div>
{#if canSetRelationshipSchemas} {#if hasLinkColumns}
<Popover <Popover
on:close={() => (relationshipFieldName = null)} on:close={() => (relationshipFieldName = null)}
open={relationshipFieldName} open={relationshipFieldName}

8
packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
View File

@ -10,8 +10,6 @@
import { getContext } from "svelte" import { getContext } from "svelte"
import { ActionButton } from "@budibase/bbui" import { ActionButton } from "@budibase/bbui"
import ColumnsSettingContent from "./ColumnsSettingContent.svelte" import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
import { isEnabled } from "helpers/featureFlags"
import { FeatureFlag } from "@budibase/types"
import DetailPopover from "components/common/DetailPopover.svelte" import DetailPopover from "components/common/DetailPopover.svelte"
const { tableColumns, datasource } = getContext("grid") const { tableColumns, datasource } = getContext("grid")
@ -46,9 +44,5 @@
{text} {text}
</ActionButton> </ActionButton>
</svelte:fragment> </svelte:fragment>
<ColumnsSettingContent <ColumnsSettingContent columns={$tableColumns} {permissions} />
columns={$tableColumns}
canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
{permissions}
/>
</DetailPopover> </DetailPopover>

21
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -371,6 +371,7 @@
delete editableColumn.relationshipType delete editableColumn.relationshipType
delete editableColumn.formulaType delete editableColumn.formulaType
delete editableColumn.constraints delete editableColumn.constraints
delete editableColumn.responseType
// Add in defaults and initial definition // Add in defaults and initial definition
const definition = fieldDefinitions[type?.toUpperCase()] const definition = fieldDefinitions[type?.toUpperCase()]
@ -386,6 +387,7 @@
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FieldType.FORMULA) { } else if (editableColumn.type === FieldType.FORMULA) {
editableColumn.formulaType = "dynamic" editableColumn.formulaType = "dynamic"
editableColumn.responseType = field.responseType || FIELDS.STRING.type
} }
} }
@ -767,6 +769,25 @@
</div> </div>
</div> </div>
{/if} {/if}
<div class="split-label">
<div class="label-length">
<Label size="M">Response Type</Label>
</div>
<div class="input-length">
<Select
bind:value={editableColumn.responseType}
options={[
FIELDS.STRING,
FIELDS.NUMBER,
FIELDS.BOOLEAN,
FIELDS.DATETIME,
]}
getOptionLabel={option => option.name}
getOptionValue={option => option.type}
tooltip="Formulas by default will return a string - however if you need another type the response can be coerced."
/>
</div>
</div>
<div class="split-label"> <div class="split-label">
<div class="label-length"> <div class="label-length">
<Label size="M">Formula</Label> <Label size="M">Formula</Label>

4
packages/builder/src/components/common/NavItem.svelte
View File

@ -84,8 +84,8 @@
on:mouseleave on:mouseleave
on:click={onClick} on:click={onClick}
on:contextmenu on:contextmenu
ondragover="return false" on:dragover={e => e.preventDefault()}
ondragenter="return false" on:dragenter={e => e.preventDefault()}
{id} {id}
{style} {style}
{draggable} {draggable}

4
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentScrollWrapper.svelte
View File

@ -68,8 +68,8 @@
on:scroll on:scroll
bind:this={scrollRef} bind:this={scrollRef}
on:drop={onDrop} on:drop={onDrop}
ondragover="return false" on:dragover={e => e.preventDefault()}
ondragenter="return false" on:dragenter={e => e.preventDefault()}
> >
<slot /> <slot />
</div> </div>

3
packages/client/package.json
View File

@ -33,8 +33,7 @@
"sanitize-html": "^2.13.0", "sanitize-html": "^2.13.0",
"screenfull": "^6.0.1", "screenfull": "^6.0.1",
"shortid": "^2.2.15", "shortid": "^2.2.15",
"svelte-spa-router": "^4.0.1", "svelte-spa-router": "^4.0.1"
"atrament": "^4.3.0"
}, },
"devDependencies": { "devDependencies": {
"@rollup/plugin-alias": "^5.1.0", "@rollup/plugin-alias": "^5.1.0",

27
packages/client/src/utils/blocks.js
View File

@ -1,5 +1,7 @@
import { makePropSafe as safe } from "@budibase/string-templates" import { makePropSafe as safe } from "@budibase/string-templates"
import { API } from "../api/index.js" import { API } from "../api/index.js"
import { UILogicalOperator } from "@budibase/types"
import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
// Map of data types to component types for search fields inside blocks // Map of data types to component types for search fields inside blocks
const schemaComponentMap = { const schemaComponentMap = {
@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
* @param formId the ID of the form containing the search fields * @param formId the ID of the form containing the search fields
*/ */
export const enrichFilter = (filter, columns, formId) => { export const enrichFilter = (filter, columns, formId) => {
let enrichedFilter = [...(filter || [])] if (!columns?.length) {
return filter
}
let newFilters = []
columns?.forEach(column => { columns?.forEach(column => {
const safePath = column.name.split(".").map(safe).join(".") const safePath = column.name.split(".").map(safe).join(".")
const stringType = column.type === "string" || column.type === "formula" const stringType = column.type === "string" || column.type === "formula"
@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For dates, use a range of the entire day selected // For dates, use a range of the entire day selected
if (dateType) { if (dateType) {
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: "rangeLow", operator: "rangeLow",
@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
const format = "YYYY-MM-DDTHH:mm:ss.SSSZ" const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}` let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
hbs = `{{#if ${binding} }}${hbs}{{/if}}` hbs = `{{#if ${binding} }}${hbs}{{/if}}`
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: "rangeHigh", operator: "rangeHigh",
@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For other fields, do an exact match // For other fields, do an exact match
else { else {
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: stringType ? "string" : "equal", operator: stringType ? "string" : "equal",
@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
}) })
} }
}) })
return enrichedFilter
return {
logicalOperator: UILogicalOperator.ALL,
onEmptyFilter: OnEmptyFilter.RETURN_ALL,
groups: [
...(filter?.groups || []),
{
logicalOperator: UILogicalOperator.ALL,
filters: newFilters,
},
],
}
} }

18
packages/frontend-core/src/components/grid/cells/FormulaCell.svelte
View File

@ -1,5 +1,21 @@
<script> <script>
import TextCell from "./TextCell.svelte" import TextCell from "./TextCell.svelte"
import DateCell from "./DateCell.svelte"
import NumberCell from "./NumberCell.svelte"
import BooleanCell from "./BooleanCell.svelte"
import { FieldType } from "@budibase/types"
export let schema
$: responseType = schema.responseType
</script> </script>
<TextCell {...$$props} readonly /> {#if responseType === FieldType.NUMBER}
<NumberCell {...$$props} readonly />
{:else if responseType === FieldType.BOOLEAN}
<BooleanCell {...$$props} readonly />
{:else if responseType === FieldType.DATETIME}
<DateCell {...$$props} readonly />
{:else}
<TextCell {...$$props} readonly />
{/if}

2
packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte
View File

@ -53,6 +53,7 @@
on:close={close} on:close={close}
maxHeight={null} maxHeight={null}
resizable resizable
minWidth={360}
> >
<div class="content"> <div class="content">
<slot /> <slot />
@ -80,7 +81,6 @@
} }
.content { .content {
width: 300px;
padding: 20px; padding: 20px;
display: flex; display: flex;
flex-direction: column; flex-direction: column;

1
packages/frontend-core/src/fetch/NestedProviderFetch.js
View File

@ -5,6 +5,7 @@ export default class NestedProviderFetch extends DataFetch {
// Nested providers should already have exposed their own schema // Nested providers should already have exposed their own schema
return { return {
schema: datasource?.value?.schema, schema: datasource?.value?.schema,
primaryDisplay: datasource?.value?.primaryDisplay,
} }
} }

2
packages/pro

@ -1 +1 @@
Subproject commit 80770215c6159e4d47f3529fd02e74bc8ad07543 Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd

41
packages/server/package.json
View File

@ -13,6 +13,7 @@
"build": "node ./scripts/build.js", "build": "node ./scripts/build.js",
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/", "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
"check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
"check:dependencies": "node ../../scripts/depcheck.js",
"build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets", "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets",
"build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers", "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
"build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson", "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
@ -49,9 +50,11 @@
"license": "GPL-3.0", "license": "GPL-3.0",
"dependencies": { "dependencies": {
"@apidevtools/swagger-parser": "10.0.3", "@apidevtools/swagger-parser": "10.0.3",
"@azure/msal-node": "^2.5.1",
"@budibase/backend-core": "0.0.0", "@budibase/backend-core": "0.0.0",
"@budibase/client": "0.0.0", "@budibase/client": "0.0.0",
"@budibase/frontend-core": "0.0.0", "@budibase/frontend-core": "0.0.0",
"@budibase/nano": "10.1.5",
"@budibase/pro": "0.0.0", "@budibase/pro": "0.0.0",
"@budibase/shared-core": "0.0.0", "@budibase/shared-core": "0.0.0",
"@budibase/string-templates": "0.0.0", "@budibase/string-templates": "0.0.0",
@ -60,15 +63,17 @@
"@bull-board/koa": "5.10.2", "@bull-board/koa": "5.10.2",
"@elastic/elasticsearch": "7.10.0", "@elastic/elasticsearch": "7.10.0",
"@google-cloud/firestore": "7.8.0", "@google-cloud/firestore": "7.8.0",
"@koa/router": "8.0.8", "@koa/cors": "5.0.0",
"@koa/router": "13.1.0",
"@socket.io/redis-adapter": "^8.2.1", "@socket.io/redis-adapter": "^8.2.1",
"@types/xml2js": "^0.4.14", "@types/xml2js": "^0.4.14",
"airtable": "0.12.2", "airtable": "0.12.2",
"arangojs": "7.2.0", "arangojs": "7.2.0",
"archiver": "7.0.1", "archiver": "7.0.1",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bson": "^6.9.0",
"buffer": "6.0.3", "buffer": "6.0.3",
"bull": "4.10.1", "bull": "4.10.1",
"chokidar": "3.5.3", "chokidar": "3.5.3",
@ -76,17 +81,20 @@
"cookies": "0.8.0", "cookies": "0.8.0",
"csvtojson": "2.0.10", "csvtojson": "2.0.10",
"curlconverter": "3.21.0", "curlconverter": "3.21.0",
"dd-trace": "5.2.0", "dayjs": "^1.10.8",
"dd-trace": "5.26.0",
"dotenv": "8.2.0", "dotenv": "8.2.0",
"form-data": "4.0.0", "form-data": "4.0.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"isolated-vm": "^4.7.2", "isolated-vm": "^4.7.2",
"jimp": "0.22.12", "jimp": "1.1.4",
"joi": "17.6.0", "joi": "17.6.0",
"js-yaml": "4.1.0", "js-yaml": "4.1.0",
"jsonschema": "1.4.0", "jsonschema": "1.4.0",
"jsonwebtoken": "9.0.2",
"knex": "2.4.2", "knex": "2.4.2",
"koa": "2.13.4", "koa": "2.13.4",
"koa-body": "4.2.0", "koa-body": "4.2.0",
@ -97,7 +105,7 @@
"lodash": "4.17.21", "lodash": "4.17.21",
"memorystream": "0.3.1", "memorystream": "0.3.1",
"mongodb": "6.7.0", "mongodb": "6.7.0",
"mssql": "10.0.1", "mssql": "11.0.1",
"mysql2": "3.9.8", "mysql2": "3.9.8",
"node-fetch": "2.6.7", "node-fetch": "2.6.7",
"object-sizeof": "2.6.1", "object-sizeof": "2.6.1",
@ -105,24 +113,28 @@
"openapi-types": "9.3.1", "openapi-types": "9.3.1",
"oracledb": "6.5.1", "oracledb": "6.5.1",
"pg": "8.10.0", "pg": "8.10.0",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-all-dbs": "1.1.1", "pouchdb-all-dbs": "1.1.1",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redis": "4", "redis": "4",
"semver": "^7.5.4",
"serialize-error": "^7.0.1", "serialize-error": "^7.0.1",
"server-destroy": "1.0.1", "server-destroy": "1.0.1",
"snowflake-promise": "^4.5.0", "snowflake-sdk": "^1.15.0",
"socket.io": "4.7.5", "socket.io": "4.8.1",
"svelte": "^4.2.10",
"tar": "6.2.1", "tar": "6.2.1",
"tmp": "0.2.3", "tmp": "0.2.3",
"to-json-schema": "0.2.5", "to-json-schema": "0.2.5",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",
"worker-farm": "1.7.0", "worker-farm": "1.7.0",
"xml2js": "0.5.0" "xml2js": "0.6.2"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.22.5",
"@babel/preset-env": "7.16.11", "@babel/preset-env": "7.16.11",
"@jest/types": "^29.6.3",
"@swc/core": "1.3.71", "@swc/core": "1.3.71",
"@swc/jest": "0.2.27", "@swc/jest": "0.2.27",
"@types/archiver": "6.0.2", "@types/archiver": "6.0.2",
@ -130,19 +142,24 @@
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/koa": "2.13.4", "@types/koa": "2.13.4",
"@types/koa-send": "^4.1.6", "@types/koa-send": "^4.1.6",
"@types/koa__router": "8.0.8", "@types/koa__cors": "5.0.0",
"@types/koa__router": "12.0.4",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/mssql": "9.1.4", "@types/mssql": "9.1.5",
"@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/oracledb": "6.5.1", "@types/oracledb": "6.5.1",
"@types/pg": "8.6.6", "@types/pg": "8.6.6",
"@types/pouchdb": "6.4.2",
"@types/server-destroy": "1.0.1", "@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.14", "@types/supertest": "2.0.14",
"@types/tar": "6.1.5", "@types/tar": "6.1.5",
"@types/tmp": "0.2.6", "@types/tmp": "0.2.6",
"@types/uuid": "8.3.4", "@types/uuid": "8.3.4",
"chance": "^1.1.12",
"copyfiles": "2.4.1", "copyfiles": "2.4.1",
"docker-compose": "0.23.17", "docker-compose": "0.23.17",
"ioredis-mock": "8.9.0",
"jest": "29.7.0", "jest": "29.7.0",
"jest-extended": "^4.0.2", "jest-extended": "^4.0.2",
"jest-openapi": "0.14.2", "jest-openapi": "0.14.2",

6
packages/server/scripts/test.sh
View File

@ -1,12 +1,12 @@
#!/bin/bash #!/bin/bash
set -e set -ex
if [[ -n $CI ]] if [[ -n $CI ]]
then then
export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS" export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail $@ jest --coverage --maxWorkers=4 --forceExit --workerIdleMemoryLimit=2000MB --bail "$@"
else else
# --maxWorkers performs better in development # --maxWorkers performs better in development
export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS" export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
jest --coverage --maxWorkers=2 --forceExit $@ jest --coverage --maxWorkers=2 --forceExit "$@"
fi fi

1239
packages/server/specs/openapi.json
View File

File diff suppressed because it is too large Load Diff

865
packages/server/specs/openapi.yaml
View File

@ -23,6 +23,13 @@ components:
description: The ID of the table which this request is targeting. description: The ID of the table which this request is targeting.
schema: schema:
type: string type: string
viewId:
in: path
name: viewId
required: true
description: The ID of the view which this request is targeting.
schema:
type: string
rowId: rowId:
in: path in: path
name: rowId name: rowId
@ -36,7 +43,7 @@ components:
required: true required: true
description: The ID of the app which this request is targeting. description: The ID of the app which this request is targeting.
schema: schema:
default: "{{ appId }}" default: "{{appId}}"
type: string type: string
appIdUrl: appIdUrl:
in: path in: path
@ -44,7 +51,7 @@ components:
required: true required: true
description: The ID of the app which this request is targeting. description: The ID of the app which this request is targeting.
schema: schema:
default: "{{ appId }}" default: "{{appId}}"
type: string type: string
queryId: queryId:
in: path in: path
@ -442,6 +449,74 @@ components:
# TYPE budibase_quota_limit_automations gauge # TYPE budibase_quota_limit_automations gauge
budibase_quota_limit_automations 9007199254740991 budibase_quota_limit_automations 9007199254740991
view:
value:
data:
name: peopleView
tableId: ta_896a325f7e8147d2a2cda93c5d236511
schema:
name:
visible: true
readonly: false
order: 1
width: 300
age:
visible: true
readonly: true
order: 2
width: 200
salary:
visible: false
readonly: false
query:
logicalOperator: all
onEmptyFilter: none
groups:
- logicalOperator: any
filters:
- operator: string
field: name
value: John
- operator: range
field: age
value:
low: 18
high: 100
primaryDisplay: name
views:
value:
data:
- name: peopleView
tableId: ta_896a325f7e8147d2a2cda93c5d236511
schema:
name:
visible: true
readonly: false
order: 1
width: 300
age:
visible: true
readonly: true
order: 2
width: 200
salary:
visible: false
readonly: false
query:
logicalOperator: all
onEmptyFilter: none
groups:
- logicalOperator: any
filters:
- operator: string
field: name
value: John
- operator: range
field: age
value:
low: 18
high: 100
primaryDisplay: name
securitySchemes: securitySchemes:
ApiKeyAuth: ApiKeyAuth:
type: apiKey type: apiKey
@ -761,7 +836,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -931,7 +1005,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -1108,7 +1181,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -1704,6 +1776,644 @@ components:
- userIds - userIds
required: required:
- data - data
view:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able to
access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a calculation is
configured all non-calculation columns will be used for
grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
viewOutput:
type: object
properties:
data:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
- id
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able
to access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a calculation
is configured all non-calculation columns will be used
for grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
id:
description: The ID of the view.
type: string
required:
- data
viewSearch:
type: object
properties:
data:
type: array
items:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
- id
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able
to access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a
calculation is configured all non-calculation
columns will be used for grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
id:
description: The ID of the view.
type: string
required:
- data
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
paths: paths:
@ -2136,6 +2846,32 @@ paths:
examples: examples:
search: search:
$ref: "#/components/examples/rows" $ref: "#/components/examples/rows"
"/views/{viewId}/rows/search":
post:
operationId: rowViewSearch
summary: Search for rows in a view
tags:
- rows
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/rowSearch"
responses:
"200":
description: The response will contain an array of rows that match the search
parameters.
content:
application/json:
schema:
$ref: "#/components/schemas/searchOutput"
examples:
search:
$ref: "#/components/examples/rows"
/tables: /tables:
post: post:
operationId: tableCreate operationId: tableCreate
@ -2359,4 +3095,123 @@ paths:
examples: examples:
users: users:
$ref: "#/components/examples/users" $ref: "#/components/examples/users"
/views:
post:
operationId: viewCreate
summary: Create a view
description: Create a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/appId"
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/view"
examples:
view:
$ref: "#/components/examples/view"
responses:
"200":
description: Returns the created view, including the ID which has been generated
for it.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
"/views/{viewId}":
put:
operationId: viewUpdate
summary: Update a view
description: Update a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/view"
examples:
view:
$ref: "#/components/examples/view"
responses:
"200":
description: Returns the updated view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
delete:
operationId: viewDestroy
summary: Delete a view
description: Delete a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
responses:
"200":
description: Returns the deleted view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
get:
operationId: viewGetById
summary: Retrieve a view
description: Lookup a view, this could be internal or external.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
responses:
"200":
description: Returns the retrieved view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
/views/search:
post:
operationId: viewSearch
summary: Search for views
description: Based on view properties (currently only name) search for views.
tags:
- views
parameters:
- $ref: "#/components/parameters/appId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/nameSearch"
responses:
"200":
description: Returns the found views, based on the search parameters.
content:
application/json:
schema:
$ref: "#/components/schemas/viewSearch"
examples:
views:
$ref: "#/components/examples/views"
tags: [] tags: []

10
packages/server/specs/parameters.ts
View File

@ -8,6 +8,16 @@ export const tableId = {
}, },
} }
export const viewId = {
in: "path",
name: "viewId",
required: true,
description: "The ID of the view which this request is targeting.",
schema: {
type: "string",
},
}
export const rowId = { export const rowId = {
in: "path", in: "path",
name: "rowId", name: "rowId",

3
packages/server/specs/resources/index.ts
View File

@ -6,6 +6,7 @@ import user from "./user"
import metrics from "./metrics" import metrics from "./metrics"
import misc from "./misc" import misc from "./misc"
import roles from "./roles" import roles from "./roles"
import view from "./view"
export const examples = { export const examples = {
...application.getExamples(), ...application.getExamples(),
@ -16,6 +17,7 @@ export const examples = {
...misc.getExamples(), ...misc.getExamples(),
...metrics.getExamples(), ...metrics.getExamples(),
...roles.getExamples(), ...roles.getExamples(),
...view.getExamples(),
} }
export const schemas = { export const schemas = {
@ -26,4 +28,5 @@ export const schemas = {
...user.getSchemas(), ...user.getSchemas(),
...misc.getSchemas(), ...misc.getSchemas(),
...roles.getSchemas(), ...roles.getSchemas(),
...view.getSchemas(),
} }

182
packages/server/specs/resources/misc.ts
View File

@ -1,99 +1,101 @@
import { object } from "./utils" import { object } from "./utils"
import Resource from "./utils/Resource" import Resource from "./utils/Resource"
export const searchSchema = {
type: "object",
properties: {
allOr: {
type: "boolean",
description:
"Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
},
string: {
type: "object",
example: {
columnName1: "value",
columnName2: "value",
},
description:
"A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
additionalProperties: {
type: "string",
description: "The value to search for in the column.",
},
},
fuzzy: {
type: "object",
description:
"Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
},
range: {
type: "object",
description:
'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
example: {
columnName1: {
low: 10,
high: 20,
},
},
},
equal: {
type: "object",
description:
"Searches for rows that have a column value that is exactly the value set.",
},
notEqual: {
type: "object",
description:
"Searches for any row which does not contain the specified column value.",
},
empty: {
type: "object",
description:
"Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
example: {
columnName1: "",
},
},
notEmpty: {
type: "object",
description: "Searches for rows which have the specified column.",
},
oneOf: {
type: "object",
description:
"Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
},
contains: {
type: "object",
description:
"Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
example: {
arrayColumn: ["a", "b"],
},
},
notContains: {
type: "object",
description:
"The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
example: {
arrayColumn: ["a", "b"],
},
},
containsAny: {
type: "object",
description:
"As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
example: {
arrayColumn: ["a", "b"],
},
},
},
}
export default new Resource().setSchemas({ export default new Resource().setSchemas({
rowSearch: object( rowSearch: object(
{ {
query: { query: searchSchema,
type: "object",
properties: {
allOr: {
type: "boolean",
description:
"Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
},
string: {
type: "object",
example: {
columnName1: "value",
columnName2: "value",
},
description:
"A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
additionalProperties: {
type: "string",
description: "The value to search for in the column.",
},
},
fuzzy: {
type: "object",
description:
"Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
},
range: {
type: "object",
description:
'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
example: {
columnName1: {
low: 10,
high: 20,
},
},
},
equal: {
type: "object",
description:
"Searches for rows that have a column value that is exactly the value set.",
},
notEqual: {
type: "object",
description:
"Searches for any row which does not contain the specified column value.",
},
empty: {
type: "object",
description:
"Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
example: {
columnName1: "",
},
},
notEmpty: {
type: "object",
description: "Searches for rows which have the specified column.",
},
oneOf: {
type: "object",
description:
"Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
},
contains: {
type: "object",
description:
"Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
example: {
arrayColumn: ["a", "b"],
},
},
notContains: {
type: "object",
description:
"The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
example: {
arrayColumn: ["a", "b"],
},
},
containsAny: {
type: "object",
description:
"As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
example: {
arrayColumn: ["a", "b"],
},
},
},
},
paginate: { paginate: {
type: "boolean", type: "boolean",
description: "Enables pagination, by default this is disabled.", description: "Enables pagination, by default this is disabled.",

274
packages/server/specs/resources/view.ts Normal file
View File

@ -0,0 +1,274 @@
import { object } from "./utils"
import Resource from "./utils/Resource"
import {
ArrayOperator,
BasicOperator,
CalculationType,
RangeOperator,
SortOrder,
SortType,
} from "@budibase/types"
import { cloneDeep } from "lodash"
const view = {
name: "peopleView",
tableId: "ta_896a325f7e8147d2a2cda93c5d236511",
schema: {
name: {
visible: true,
readonly: false,
order: 1,
width: 300,
},
age: {
visible: true,
readonly: true,
order: 2,
width: 200,
},
salary: {
visible: false,
readonly: false,
},
},
query: {
logicalOperator: "all",
onEmptyFilter: "none",
groups: [
{
logicalOperator: "any",
filters: [
{ operator: "string", field: "name", value: "John" },
{ operator: "range", field: "age", value: { low: 18, high: 100 } },
],
},
],
},
primaryDisplay: "name",
}
const baseColumnDef = {
visible: {
type: "boolean",
description:
"Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it.",
},
readonly: {
type: "boolean",
description:
"When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated.",
},
order: {
type: "integer",
description:
"A number defining where the column shows up in tables, lowest being first.",
},
width: {
type: "integer",
description:
"A width for the column, defined in pixels - this affects rendering in tables.",
},
column: {
type: "array",
description:
"If this is a relationship column, we can set the columns we wish to include",
items: {
type: "object",
properties: {
readonly: {
type: "boolean",
},
},
},
},
}
const logicalOperator = {
description:
"When using groups this defines whether all of the filters must match, or only one of them.",
type: "string",
enum: ["all", "any"],
}
const filterGroup = {
description: "A grouping of filters to be applied.",
type: "array",
items: {
type: "object",
properties: {
logicalOperator,
filters: {
description: "A list of filters to apply",
type: "array",
items: {
type: "object",
properties: {
operator: {
type: "string",
description:
"The type of search operation which is being performed.",
enum: [
...Object.values(BasicOperator),
...Object.values(ArrayOperator),
...Object.values(RangeOperator),
],
},
field: {
type: "string",
description: "The field in the view to perform the search on.",
},
value: {
description:
"The value to search for - the type will depend on the operator in use.",
oneOf: [
{ type: "string" },
{ type: "number" },
{ type: "boolean" },
{ type: "object" },
{ type: "array" },
],
},
},
},
},
},
},
}
// have to clone to avoid constantly recursive structure - we can't represent this easily
const layeredFilterGroup: any = cloneDeep(filterGroup)
layeredFilterGroup.items.properties.groups = filterGroup
const viewQuerySchema = {
description: "Search parameters for view",
type: "object",
required: [],
properties: {
logicalOperator,
onEmptyFilter: {
description:
"If no filters match, should the view return all rows, or no rows.",
type: "string",
enum: ["all", "none"],
},
groups: layeredFilterGroup,
},
}
const viewSchema = {
description: "The view to be created/updated.",
type: "object",
required: ["name", "schema", "tableId"],
properties: {
name: {
description: "The name of the view.",
type: "string",
},
tableId: {
description: "The ID of the table this view is based on.",
type: "string",
},
type: {
description: "The type of view - standard (empty value) or calculation.",
type: "string",
enum: ["calculation"],
},
primaryDisplay: {
type: "string",
description:
"A column used to display rows from this view - usually used when rendered in tables.",
},
query: viewQuerySchema,
sort: {
type: "object",
required: ["field"],
properties: {
field: {
type: "string",
description: "The field from the table/view schema to sort on.",
},
order: {
type: "string",
description: "The order in which to sort.",
enum: Object.values(SortOrder),
},
type: {
type: "string",
description:
"The type of sort to perform (by number, or by alphabetically).",
enum: Object.values(SortType),
},
},
},
schema: {
type: "object",
additionalProperties: {
oneOf: [
{
type: "object",
properties: baseColumnDef,
},
{
type: "object",
properties: {
calculationType: {
type: "string",
description:
"This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.",
enum: Object.values(CalculationType),
},
field: {
type: "string",
description:
"The field from the table to perform the calculation on.",
},
distinct: {
type: "boolean",
description:
"Can be used in tandem with the count calculation type, to count unique entries.",
},
},
},
],
},
},
},
}
const viewOutputSchema = {
...viewSchema,
properties: {
...viewSchema.properties,
id: {
description: "The ID of the view.",
type: "string",
},
},
required: [...viewSchema.required, "id"],
}
export default new Resource()
.setExamples({
view: {
value: {
data: view,
},
},
views: {
value: {
data: [view],
},
},
})
.setSchemas({
view: viewSchema,
viewOutput: object({
data: viewOutputSchema,
}),
viewSearch: object({
data: {
type: "array",
items: viewOutputSchema,
},
}),
})

3
packages/server/src/api/controllers/public/mapping/applications.ts
View File

@ -1,6 +1,7 @@
import { Application } from "./types" import { Application } from "./types"
import { RequiredKeys } from "@budibase/types"
function application(body: any): Application { function application(body: any): RequiredKeys<Application> {
let app = body?.application ? body.application : body let app = body?.application ? body.application : body
return { return {
_id: app.appId, _id: app.appId,

2
packages/server/src/api/controllers/public/mapping/index.ts
View File

@ -3,6 +3,7 @@ import applications from "./applications"
import users from "./users" import users from "./users"
import rows from "./rows" import rows from "./rows"
import queries from "./queries" import queries from "./queries"
import views from "./views"
export default { export default {
...tables, ...tables,
@ -10,4 +11,5 @@ export default {
...users, ...users,
...rows, ...rows,
...queries, ...queries,
...views,
} }

3
packages/server/src/api/controllers/public/mapping/queries.ts
View File

@ -1,6 +1,7 @@
import { Query, ExecuteQuery } from "./types" import { Query, ExecuteQuery } from "./types"
import { RequiredKeys } from "@budibase/types"
function query(body: any): Query { function query(body: any): RequiredKeys<Query> {
return { return {
_id: body._id, _id: body._id,
datasourceId: body.datasourceId, datasourceId: body.datasourceId,

3
packages/server/src/api/controllers/public/mapping/rows.ts
View File

@ -1,6 +1,7 @@
import { Row, RowSearch } from "./types" import { Row, RowSearch } from "./types"
import { RequiredKeys } from "@budibase/types"
function row(body: any): Row { function row(body: any): RequiredKeys<Row> {
delete body._rev delete body._rev
// have to input everything, since structure unknown // have to input everything, since structure unknown
return { return {

3
packages/server/src/api/controllers/public/mapping/tables.ts
View File

@ -1,6 +1,7 @@
import { Table } from "./types" import { Table } from "./types"
import { RequiredKeys } from "@budibase/types"
function table(body: any): Table { function table(body: any): RequiredKeys<Table> {
return { return {
_id: body._id, _id: body._id,
name: body.name, name: body.name,

3
packages/server/src/api/controllers/public/mapping/types.ts
View File

@ -9,6 +9,9 @@ export type CreateApplicationParams = components["schemas"]["application"]
export type Table = components["schemas"]["tableOutput"]["data"] export type Table = components["schemas"]["tableOutput"]["data"]
export type CreateTableParams = components["schemas"]["table"] export type CreateTableParams = components["schemas"]["table"]
export type View = components["schemas"]["viewOutput"]["data"]
export type CreateViewParams = components["schemas"]["view"]
export type Row = components["schemas"]["rowOutput"]["data"] export type Row = components["schemas"]["rowOutput"]["data"]
export type RowSearch = components["schemas"]["searchOutput"] export type RowSearch = components["schemas"]["searchOutput"]
export type CreateRowParams = components["schemas"]["row"] export type CreateRowParams = components["schemas"]["row"]

3
packages/server/src/api/controllers/public/mapping/users.ts
View File

@ -1,6 +1,7 @@
import { User } from "./types" import { User } from "./types"
import { RequiredKeys } from "@budibase/types"
function user(body: any): User { function user(body: any): RequiredKeys<User> {
return { return {
_id: body._id, _id: body._id,
email: body.email, email: body.email,

32
packages/server/src/api/controllers/public/mapping/views.ts Normal file
View File

@ -0,0 +1,32 @@
import { View } from "./types"
import { ViewV2, Ctx, RequiredKeys } from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
function view(body: ViewV2): RequiredKeys<View> {
return {
id: body.id,
tableId: body.tableId,
type: body.type,
name: body.name,
schema: body.schema!,
primaryDisplay: body.primaryDisplay,
query: dataFilters.buildQuery(body.query),
sort: body.sort,
}
}
function mapView(ctx: Ctx<{ data: ViewV2 }>): { data: View } {
return {
data: view(ctx.body.data),
}
}
function mapViews(ctx: Ctx<{ data: ViewV2[] }>): { data: View[] } {
const views = ctx.body.data.map((body: ViewV2) => view(body))
return { data: views }
}
export default {
mapView,
mapViews,
}

18
packages/server/src/api/controllers/public/rows.ts
View File

@ -22,13 +22,13 @@ export function fixRow(row: Row, params: any) {
return row return row
} }
export async function search(ctx: UserCtx, next: Next) { function buildSearchRequestBody(ctx: UserCtx) {
let { sort, paginate, bookmark, limit, query } = ctx.request.body let { sort, paginate, bookmark, limit, query } = ctx.request.body
// update the body to the correct format of the internal search // update the body to the correct format of the internal search
if (!sort) { if (!sort) {
sort = {} sort = {}
} }
ctx.request.body = { return {
sort: sort.column, sort: sort.column,
sortType: sort.type, sortType: sort.type,
sortOrder: sort.order, sortOrder: sort.order,
@ -37,10 +37,23 @@ export async function search(ctx: UserCtx, next: Next) {
limit, limit,
query, query,
} }
}
export async function search(ctx: UserCtx, next: Next) {
ctx.request.body = buildSearchRequestBody(ctx)
await rowController.search(ctx) await rowController.search(ctx)
await next() await next()
} }
export async function viewSearch(ctx: UserCtx, next: Next) {
ctx.request.body = buildSearchRequestBody(ctx)
ctx.params = {
viewId: ctx.params.viewId,
}
await rowController.views.searchView(ctx)
await next()
}
export async function create(ctx: UserCtx, next: Next) { export async function create(ctx: UserCtx, next: Next) {
ctx.request.body = fixRow(ctx.request.body, ctx.params) ctx.request.body = fixRow(ctx.request.body, ctx.params)
await rowController.save(ctx) await rowController.save(ctx)
@ -79,4 +92,5 @@ export default {
update, update,
destroy, destroy,
search, search,
viewSearch,
} }

95
packages/server/src/api/controllers/public/views.ts Normal file
View File

@ -0,0 +1,95 @@
import { search as stringSearch } from "./utils"
import * as controller from "../view"
import { ViewV2, UserCtx, UISearchFilter, PublicAPIView } from "@budibase/types"
import { Next } from "koa"
import { merge } from "lodash"
function viewRequest(view: PublicAPIView, params?: { viewId: string }) {
const viewV2: ViewV2 = view
if (!viewV2) {
return viewV2
}
if (params?.viewId) {
viewV2.id = params.viewId
}
if (!view.query) {
viewV2.query = {}
} else {
// public API only has one form of query
viewV2.queryUI = viewV2.query as UISearchFilter
}
viewV2.version = 2
return viewV2
}
function viewResponse(view: ViewV2): PublicAPIView {
// remove our internal structure - always un-necessary
delete view.query
return {
...view,
query: view.queryUI,
}
}
function viewsResponse(views: ViewV2[]): PublicAPIView[] {
return views.map(viewResponse)
}
export async function search(ctx: UserCtx, next: Next) {
const { name } = ctx.request.body
await controller.v2.fetch(ctx)
ctx.body.data = viewsResponse(stringSearch(ctx.body.data, name))
await next()
}
export async function create(ctx: UserCtx, next: Next) {
ctx = merge(ctx, {
request: {
body: viewRequest(ctx.request.body),
},
})
await controller.v2.create(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function read(ctx: UserCtx, next: Next) {
ctx = merge(ctx, {
params: {
viewId: ctx.params.viewId,
},
})
await controller.v2.get(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function update(ctx: UserCtx, next: Next) {
const viewId = ctx.params.viewId
ctx = merge(ctx, {
request: {
body: {
data: viewRequest(ctx.request.body, { viewId }),
},
},
params: {
viewId,
},
})
await controller.v2.update(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function destroy(ctx: UserCtx, next: Next) {
await controller.v2.remove(ctx)
await next()
}
export default {
create,
read,
update,
destroy,
search,
}

5
packages/server/src/api/controllers/query/import/sources/curl.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url"
const curlconverter = require("curlconverter") const curlconverter = require("curlconverter")
const parseCurl = (data: string): any => { const parseCurl = (data: string): Promise<any> => {
const curlJson = curlconverter.toJsonString(data) const curlJson = curlconverter.toJsonString(data)
return JSON.parse(curlJson) return JSON.parse(curlJson)
} }
@ -53,8 +53,7 @@ export class Curl extends ImportSource {
isSupported = async (data: string): Promise<boolean> => { isSupported = async (data: string): Promise<boolean> => {
try { try {
const curl = parseCurl(data) this.curl = parseCurl(data)
this.curl = curl
} catch (err) { } catch (err) {
return false return false
} }

3
packages/server/src/api/controllers/query/index.ts
View File

@ -23,6 +23,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
import { findHBSBlocks } from "@budibase/string-templates" import { findHBSBlocks } from "@budibase/string-templates"
import { ObjectId } from "mongodb"
const Runner = new Thread(ThreadType.QUERY, { const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT, timeoutMs: env.QUERY_THREAD_TIMEOUT,
@ -223,6 +224,8 @@ export async function preview(
} else { } else {
fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
} }
} else if (field instanceof ObjectId) {
fieldMetadata = makeQuerySchema(FieldType.STRING, key)
} else { } else {
fieldMetadata = makeQuerySchema(FieldType.JSON, key) fieldMetadata = makeQuerySchema(FieldType.JSON, key)
} }

1
packages/server/src/api/controllers/row/views.ts
View File

@ -50,6 +50,7 @@ export async function searchView(
result.rows.forEach(r => (r._viewId = view.id)) result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result ctx.body = result
} }
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) { if (request.sort) {
return { return {

13
packages/server/src/api/controllers/table/utils.ts
View File

@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils"
import viewTemplate from "../view/viewBuilder" import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { context, events, features, HTTPError } from "@budibase/backend-core" import { context, events, HTTPError } from "@budibase/backend-core"
import { import {
AutoFieldSubType, AutoFieldSubType,
Database, Database,
Datasource, Datasource,
FeatureFlag,
FieldSchema, FieldSchema,
FieldType, FieldType,
NumberFieldMetadata, NumberFieldMetadata,
@ -336,9 +335,8 @@ class TableSaveFunctions {
importRows: this.importRows, importRows: this.importRows,
userId: this.userId, userId: this.userId,
}) })
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.addTable(table) await sdk.tables.sqs.addTable(table)
}
return table return table
} }
@ -530,9 +528,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
if (rows) { if (rows) {
await AttachmentCleanup.tableDelete(table, rows) await AttachmentCleanup.tableDelete(table, rows)
} }
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.removeTable(table) await sdk.tables.sqs.removeTable(table)
}
} }
const _TableSaveFunctions = TableSaveFunctions const _TableSaveFunctions = TableSaveFunctions

7
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -12,6 +12,7 @@ import {
RelationSchemaField, RelationSchemaField,
ViewFieldMetadata, ViewFieldMetadata,
CalculationType, CalculationType,
ViewFetchResponseEnriched,
CountDistinctCalculationFieldMetadata, CountDistinctCalculationFieldMetadata,
CountCalculationFieldMetadata, CountCalculationFieldMetadata,
} from "@budibase/types" } from "@budibase/types"
@ -125,6 +126,12 @@ export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
} }
} }
export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) {
ctx.body = {
data: await sdk.views.getAllEnriched(),
}
}
export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) { export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const view = ctx.request.body const view = ctx.request.body
const { tableId } = view const { tableId } = view

10
packages/server/src/api/routes/public/index.ts
View File

@ -4,19 +4,21 @@ import queryEndpoints from "./queries"
import tableEndpoints from "./tables" import tableEndpoints from "./tables"
import rowEndpoints from "./rows" import rowEndpoints from "./rows"
import userEndpoints from "./users" import userEndpoints from "./users"
import viewEndpoints from "./views"
import roleEndpoints from "./roles" import roleEndpoints from "./roles"
import authorized from "../../../middleware/authorized" import authorized from "../../../middleware/authorized"
import publicApi from "../../../middleware/publicApi" import publicApi from "../../../middleware/publicApi"
import { paramResource, paramSubResource } from "../../../middleware/resourceId" import { paramResource, paramSubResource } from "../../../middleware/resourceId"
import { PermissionType, PermissionLevel } from "@budibase/types" import { PermissionLevel, PermissionType } from "@budibase/types"
import { CtxFn } from "./utils/Endpoint" import { CtxFn } from "./utils/Endpoint"
import mapperMiddleware from "./middleware/mapper" import mapperMiddleware from "./middleware/mapper"
import env from "../../../environment" import env from "../../../environment"
import { middleware, redis } from "@budibase/backend-core"
import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
import cors from "@koa/cors"
// below imports don't have declaration files // below imports don't have declaration files
const Router = require("@koa/router") const Router = require("@koa/router")
const { RateLimit, Stores } = require("koa2-ratelimit") const { RateLimit, Stores } = require("koa2-ratelimit")
import { middleware, redis } from "@budibase/backend-core"
import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
interface KoaRateLimitOptions { interface KoaRateLimitOptions {
socket: { socket: {
@ -81,6 +83,7 @@ const publicRouter = new Router({
if (limiter && !env.isDev()) { if (limiter && !env.isDev()) {
publicRouter.use(limiter) publicRouter.use(limiter)
} }
publicRouter.use(cors())
function addMiddleware( function addMiddleware(
endpoints: any, endpoints: any,
@ -149,6 +152,7 @@ applyAdminRoutes(metricEndpoints)
applyAdminRoutes(roleEndpoints) applyAdminRoutes(roleEndpoints)
applyRoutes(appEndpoints, PermissionType.APP, "appId") applyRoutes(appEndpoints, PermissionType.APP, "appId")
applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId")
applyRoutes(viewEndpoints, PermissionType.VIEW, "viewId")
applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(userEndpoints, PermissionType.USER, "userId")
applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId")
// needs to be applied last for routing purposes, don't override other endpoints // needs to be applied last for routing purposes, don't override other endpoints

30
packages/server/src/api/routes/public/middleware/mapper.ts
View File

@ -1,9 +1,10 @@
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import mapping from "../../../controllers/public/mapping" import mapping from "../../../controllers/public/mapping"
enum Resources { enum Resource {
APPLICATION = "applications", APPLICATION = "applications",
TABLES = "tables", TABLES = "tables",
VIEWS = "views",
ROWS = "rows", ROWS = "rows",
USERS = "users", USERS = "users",
QUERIES = "queries", QUERIES = "queries",
@ -15,7 +16,7 @@ function isAttachment(ctx: Ctx) {
} }
function isArrayResponse(ctx: Ctx) { function isArrayResponse(ctx: Ctx) {
return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body) return ctx.url.endsWith(Resource.SEARCH) || Array.isArray(ctx.body)
} }
function noResponse(ctx: Ctx) { function noResponse(ctx: Ctx) {
@ -38,6 +39,14 @@ function processTables(ctx: Ctx) {
} }
} }
function processViews(ctx: Ctx) {
if (isArrayResponse(ctx)) {
return mapping.mapViews(ctx)
} else {
return mapping.mapView(ctx)
}
}
function processRows(ctx: Ctx) { function processRows(ctx: Ctx) {
if (isArrayResponse(ctx)) { if (isArrayResponse(ctx)) {
return mapping.mapRowSearch(ctx) return mapping.mapRowSearch(ctx)
@ -71,20 +80,27 @@ export default async (ctx: Ctx, next: any) => {
let body = {} let body = {}
switch (urlParts[0]) { switch (urlParts[0]) {
case Resources.APPLICATION: case Resource.APPLICATION:
body = processApplications(ctx) body = processApplications(ctx)
break break
case Resources.TABLES: case Resource.TABLES:
if (urlParts[2] === Resources.ROWS) { if (urlParts[2] === Resource.ROWS) {
body = processRows(ctx) body = processRows(ctx)
} else { } else {
body = processTables(ctx) body = processTables(ctx)
} }
break break
case Resources.USERS: case Resource.VIEWS:
if (urlParts[2] === Resource.ROWS) {
body = processRows(ctx)
} else {
body = processViews(ctx)
}
break
case Resource.USERS:
body = processUsers(ctx) body = processUsers(ctx)
break break
case Resources.QUERIES: case Resource.QUERIES:
body = processQueries(ctx) body = processQueries(ctx)
break break
} }

38
packages/server/src/api/routes/public/rows.ts
View File

@ -1,4 +1,4 @@
import controller from "../../controllers/public/rows" import controller, { viewSearch } from "../../controllers/public/rows"
import Endpoint from "./utils/Endpoint" import Endpoint from "./utils/Endpoint"
import { externalSearchValidator } from "../utils/validators" import { externalSearchValidator } from "../utils/validators"
@ -168,4 +168,40 @@ read.push(
).addMiddleware(externalSearchValidator()) ).addMiddleware(externalSearchValidator())
) )
/**
* @openapi
* /views/{viewId}/rows/search:
* post:
* operationId: rowViewSearch
* summary: Search for rows in a view
* tags:
* - rows
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* requestBody:
* required: true
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/rowSearch'
* responses:
* 200:
* description: The response will contain an array of rows that match the search parameters.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/searchOutput'
* examples:
* search:
* $ref: '#/components/examples/rows'
*/
read.push(
new Endpoint(
"post",
"/views/:viewId/rows/search",
controller.viewSearch
).addMiddleware(externalSearchValidator())
)
export default { read, write } export default { read, write }

118
packages/server/src/api/routes/public/tests/Request.ts
View File

@ -1,13 +1,24 @@
import { User, Table, SearchFilters, Row } from "@budibase/types" import {
User,
Table,
SearchFilters,
Row,
ViewV2Schema,
ViewV2,
ViewV2Type,
PublicAPIView,
} from "@budibase/types"
import { HttpMethod, MakeRequestResponse, generateMakeRequest } from "./utils" import { HttpMethod, MakeRequestResponse, generateMakeRequest } from "./utils"
import TestConfiguration from "../../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
import { Expectations } from "../../../../tests/utilities/api/base"
type RequestOpts = { internal?: boolean; appId?: string } type RequestOpts = { internal?: boolean; appId?: string }
type Response<T> = { data: T }
export interface PublicAPIExpectations { export interface PublicAPIExpectations {
status?: number status?: number
body?: Record<string, any> body?: Record<string, any>
headers?: Record<string, string>
} }
export class PublicAPIRequest { export class PublicAPIRequest {
@ -15,6 +26,7 @@ export class PublicAPIRequest {
private appId: string | undefined private appId: string | undefined
tables: PublicTableAPI tables: PublicTableAPI
views: PublicViewAPI
rows: PublicRowAPI rows: PublicRowAPI
apiKey: string apiKey: string
@ -28,6 +40,7 @@ export class PublicAPIRequest {
this.appId = appId this.appId = appId
this.tables = new PublicTableAPI(this) this.tables = new PublicTableAPI(this)
this.rows = new PublicRowAPI(this) this.rows = new PublicRowAPI(this)
this.views = new PublicViewAPI(this)
} }
static async init(config: TestConfiguration, user: User, opts?: RequestOpts) { static async init(config: TestConfiguration, user: User, opts?: RequestOpts) {
@ -59,6 +72,12 @@ export class PublicAPIRequest {
if (expectations?.body) { if (expectations?.body) {
expect(res.body).toEqual(expectations?.body) expect(res.body).toEqual(expectations?.body)
} }
if (expectations?.headers) {
for (let [header, value] of Object.entries(expectations.headers)) {
const found = res.headers[header]
expect(found?.toLowerCase()).toEqual(value)
}
}
return res.body return res.body
} }
} }
@ -73,9 +92,16 @@ export class PublicTableAPI {
async create( async create(
table: Table, table: Table,
expectations?: PublicAPIExpectations expectations?: PublicAPIExpectations
): Promise<{ data: Table }> { ): Promise<Response<Table>> {
return this.request.send("post", "/tables", table, expectations) return this.request.send("post", "/tables", table, expectations)
} }
async search(
name: string,
expectations?: PublicAPIExpectations
): Promise<Response<Table[]>> {
return this.request.send("post", "/tables/search", { name }, expectations)
}
} }
export class PublicRowAPI { export class PublicRowAPI {
@ -85,11 +111,24 @@ export class PublicRowAPI {
this.request = request this.request = request
} }
async create(
tableId: string,
row: Row,
expectations?: PublicAPIExpectations
): Promise<Response<Row>> {
return this.request.send(
"post",
`/tables/${tableId}/rows`,
row,
expectations
)
}
async search( async search(
tableId: string, tableId: string,
query: SearchFilters, query: SearchFilters,
expectations?: PublicAPIExpectations expectations?: PublicAPIExpectations
): Promise<{ data: Row[] }> { ): Promise<Response<Row[]>> {
return this.request.send( return this.request.send(
"post", "post",
`/tables/${tableId}/rows/search`, `/tables/${tableId}/rows/search`,
@ -99,4 +138,75 @@ export class PublicRowAPI {
expectations expectations
) )
} }
async viewSearch(
viewId: string,
query: SearchFilters,
expectations?: PublicAPIExpectations
): Promise<Response<Row[]>> {
return this.request.send(
"post",
`/views/${viewId}/rows/search`,
{
query,
},
expectations
)
}
}
export class PublicViewAPI {
request: PublicAPIRequest
constructor(request: PublicAPIRequest) {
this.request = request
}
async create(
view: Omit<PublicAPIView, "id" | "version">,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("post", "/views", view, expectations)
}
async update(
viewId: string,
view: Omit<PublicAPIView, "id" | "version">,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("put", `/views/${viewId}`, view, expectations)
}
async destroy(
viewId: string,
expectations?: PublicAPIExpectations
): Promise<void> {
return this.request.send(
"delete",
`/views/${viewId}`,
undefined,
expectations
)
}
async find(
viewId: string,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("get", `/views/${viewId}`, undefined, expectations)
}
async search(
viewName: string,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView[]>> {
return this.request.send(
"post",
"/views/search",
{
name: viewName,
},
expectations
)
}
} }

21
packages/server/src/api/routes/public/tests/cors.spec.ts Normal file
View File

@ -0,0 +1,21 @@
import * as setup from "../../tests/utilities"
import { PublicAPIRequest } from "./Request"
describe("check public API security", () => {
const config = setup.getConfig()
let request: PublicAPIRequest
beforeAll(async () => {
await config.init()
request = await PublicAPIRequest.init(config, await config.globalUser())
})
it("should have Access-Control-Allow-Origin set to *", async () => {
await request.tables.search("", {
status: 200,
headers: {
"access-control-allow-origin": "*",
},
})
})
})

95
packages/server/src/api/routes/public/tests/views.spec.ts Normal file
View File

@ -0,0 +1,95 @@
import * as setup from "../../tests/utilities"
import { basicTable } from "../../../../tests/utilities/structures"
import { BasicOperator, Table, UILogicalOperator } from "@budibase/types"
import { PublicAPIRequest } from "./Request"
import { generator } from "@budibase/backend-core/tests"
describe("check public API security", () => {
const config = setup.getConfig()
let request: PublicAPIRequest, table: Table
beforeAll(async () => {
await config.init()
request = await PublicAPIRequest.init(config, await config.globalUser())
table = (await request.tables.create(basicTable())).data
})
function baseView() {
return {
name: generator.word(),
tableId: table._id!,
query: {},
schema: {
name: {
readonly: true,
visible: true,
},
},
}
}
it("should be able to create a view", async () => {
await request.views.create(baseView(), { status: 201 })
})
it("should be able to update a view", async () => {
const view = await request.views.create(baseView(), { status: 201 })
const response = await request.views.update(view.data.id, {
...view.data,
name: "new name",
})
})
it("should be able to search views", async () => {
const viewName = "view to search for"
const view = await request.views.create(
{
...baseView(),
name: viewName,
},
{ status: 201 }
)
const results = await request.views.search(viewName, {
status: 200,
})
expect(results.data.length).toEqual(1)
expect(results.data[0].id).toEqual(view.data.id)
})
it("should be able to delete a view", async () => {
const view = await request.views.create(baseView(), { status: 201 })
const result = await request.views.destroy(view.data.id, { status: 204 })
expect(result).toBeDefined()
})
it("should be able to search rows through a view", async () => {
const row1 = await request.rows.create(
table._id!,
{ name: "hello world" },
{ status: 200 }
)
await request.rows.create(table._id!, { name: "foo bar" }, { status: 200 })
const response = await request.views.create(
{
...baseView(),
query: {
logicalOperator: UILogicalOperator.ANY,
groups: [
{
filters: [
{
operator: BasicOperator.STRING,
field: "name",
value: "hello",
},
],
},
],
},
},
{ status: 201 }
)
const results = await request.rows.viewSearch(response.data.id, {})
expect(results.data.length).toEqual(1)
})
})

165
packages/server/src/api/routes/public/views.ts Normal file
View File

@ -0,0 +1,165 @@
import controller from "../../controllers/public/views"
import Endpoint from "./utils/Endpoint"
import { viewValidator, nameValidator } from "../utils/validators"
const read = [],
write = []
/**
* @openapi
* /views:
* post:
* operationId: viewCreate
* summary: Create a view
* description: Create a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/appId'
* requestBody:
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/view'
* examples:
* view:
* $ref: '#/components/examples/view'
* responses:
* 200:
* description: Returns the created view, including the ID which has been generated for it.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(
new Endpoint("post", "/views", controller.create).addMiddleware(
viewValidator()
)
)
/**
* @openapi
* /views/{viewId}:
* put:
* operationId: viewUpdate
* summary: Update a view
* description: Update a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* requestBody:
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/view'
* examples:
* view:
* $ref: '#/components/examples/view'
* responses:
* 200:
* description: Returns the updated view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(
new Endpoint("put", "/views/:viewId", controller.update).addMiddleware(
viewValidator()
)
)
/**
* @openapi
* /views/{viewId}:
* delete:
* operationId: viewDestroy
* summary: Delete a view
* description: Delete a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* responses:
* 200:
* description: Returns the deleted view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(new Endpoint("delete", "/views/:viewId", controller.destroy))
/**
* @openapi
* /views/{viewId}:
* get:
* operationId: viewGetById
* summary: Retrieve a view
* description: Lookup a view, this could be internal or external.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* responses:
* 200:
* description: Returns the retrieved view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
read.push(new Endpoint("get", "/views/:viewId", controller.read))
/**
* @openapi
* /views/search:
* post:
* operationId: viewSearch
* summary: Search for views
* description: Based on view properties (currently only name) search for views.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/appId'
* requestBody:
* required: true
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/nameSearch'
* responses:
* 200:
* description: Returns the found views, based on the search parameters.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewSearch'
* examples:
* views:
* $ref: '#/components/examples/views'
*/
read.push(
new Endpoint("post", "/views/search", controller.search).addMiddleware(
nameValidator()
)
)
export default { read, write }

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -16,7 +16,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities" import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils" import { AppStatus } from "../../../db/utils"
import { events, utils, context, features } from "@budibase/backend-core" import { events, utils, context } from "@budibase/backend-core"
import env from "../../../environment" import env from "../../../environment"
import { type App, BuiltinPermissionID } from "@budibase/types" import { type App, BuiltinPermissionID } from "@budibase/types"
import tk from "timekeeper" import tk from "timekeeper"
@ -355,21 +355,6 @@ describe("/applications", () => {
expect(events.app.deleted).toHaveBeenCalledTimes(1) expect(events.app.deleted).toHaveBeenCalledTimes(1)
expect(events.app.unpublished).toHaveBeenCalledTimes(1) expect(events.app.unpublished).toHaveBeenCalledTimes(1)
}) })
it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => {
const prodAppId = app.appId.replace("_dev", "")
nock("http://localhost:10000")
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
}) })
describe("POST /api/applications/:appId/duplicate", () => { describe("POST /api/applications/:appId/duplicate", () => {

32
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -19,8 +19,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import { import {
DatabaseName, DatabaseName,
getDatasource, datasourceDescribe,
knexClient,
} from "../../../integrations/tests/utils" } from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures" import { tableForDatasource } from "../../../tests/utilities/structures"
import nock from "nock" import nock from "nock"
@ -69,7 +68,7 @@ describe("/datasources", () => {
{ {
status: 500, status: 500,
body: { body: {
message: "No datasource implementation found.", message: 'No datasource implementation found called: "invalid"',
}, },
} }
) )
@ -163,21 +162,26 @@ describe("/datasources", () => {
}) })
}) })
}) })
})
describe.each([ const descriptions = datasourceDescribe({
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], })
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], if (descriptions.length) {
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], describe.each(descriptions)("$dbName", ({ config, dsProvider }) => {
])("%s", (_, dsProvider) => { let datasource: Datasource
let rawDatasource: Datasource let rawDatasource: Datasource
let client: Knex let client: Knex
beforeEach(async () => { beforeEach(async () => {
rawDatasource = await dsProvider const ds = await dsProvider()
datasource = await config.api.datasource.create(rawDatasource) rawDatasource = ds.rawDatasource!
client = await knexClient(rawDatasource) datasource = ds.datasource!
client = ds.client!
jest.clearAllMocks()
nock.cleanAll()
}) })
describe("get", () => { describe("get", () => {
@ -492,4 +496,4 @@ describe("/datasources", () => {
}) })
}) })
}) })
}) }

1775
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

File diff suppressed because it is too large Load Diff

1379
packages/server/src/api/routes/tests/queries/mongodb.spec.ts
View File

File diff suppressed because it is too large Load Diff

6628
packages/server/src/api/routes/tests/row.spec.ts
View File

File diff suppressed because it is too large Load Diff

65
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -9,15 +9,20 @@ import {
import { automations } from "@budibase/pro" import { automations } from "@budibase/pro"
import { import {
CreateRowActionRequest, CreateRowActionRequest,
Datasource,
DocumentType, DocumentType,
PermissionLevel, PermissionLevel,
RowActionResponse, RowActionResponse,
Table,
TableRowActions, TableRowActions,
} from "@budibase/types" } from "@budibase/types"
import * as setup from "./utilities" import * as setup from "./utilities"
import { generator, mocks } from "@budibase/backend-core/tests" import { generator, mocks } from "@budibase/backend-core/tests"
import { Expectations } from "../../../tests/utilities/api/base" import { Expectations } from "../../../tests/utilities/api/base"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { generateRowActionsID } from "../../../db/utils" import { generateRowActionsID } from "../../../db/utils"
const expectAutomationId = () => const expectAutomationId = () =>
@ -969,36 +974,38 @@ describe("/rowsActions", () => {
status: 200, status: 200,
}) })
}) })
})
})
it.each([ const descriptions = datasourceDescribe({
[ only: [DatabaseName.SQS, DatabaseName.POSTGRES],
"internal", })
async () => {
await config.newTenant() if (descriptions.length) {
describe.each(descriptions)(
"row actions ($dbName)",
({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource
})
async function getTable(): Promise<Table> {
if (isInternal) {
await config.api.application.addSampleData(config.getAppId()) await config.api.application.addSampleData(config.getAppId())
const tables = await config.api.table.fetch() const tables = await config.api.table.fetch()
const table = tables.find( return tables.find(t => t.sourceId === DEFAULT_BB_DATASOURCE_ID)!
t => t.sourceId === DEFAULT_BB_DATASOURCE_ID } else {
)!
return table
},
],
[
"external",
async () => {
await config.newTenant()
const ds = await config.createDatasource({
datasource: await getDatasource(DatabaseName.POSTGRES),
})
const table = await config.api.table.save( const table = await config.api.table.save(
setup.structures.tableForDatasource(ds) setup.structures.tableForDatasource(datasource!)
) )
return table return table
}, }
], }
])(
"should delete all the row actions (and automations) for its tables when a datasource is deleted", it("should delete all the row actions (and automations) for its tables when a datasource is deleted", async () => {
async (_, getTable) => {
async function getRowActionsFromDb(tableId: string) { async function getRowActionsFromDb(tableId: string) {
return await context.doInAppContext(config.getAppId(), async () => { return await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB() const db = context.getAppDB()
@ -1032,7 +1039,7 @@ describe("/rowsActions", () => {
expect(automationsResp.automations).toHaveLength(0) expect(automationsResp.automations).toHaveLength(0)
expect(await getRowActionsFromDb(tableId)).toBeUndefined() expect(await getRowActionsFromDb(tableId)).toBeUndefined()
} })
) }
}) )
}) }

7212
packages/server/src/api/routes/tests/search.spec.ts
View File

File diff suppressed because it is too large Load Diff

2778
packages/server/src/api/routes/tests/table.spec.ts
View File

File diff suppressed because it is too large Load Diff

65
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,6 @@ import * as setup from "./utilities"
import path from "path" import path from "path"
import nock from "nock" import nock from "nock"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
interface App { interface App {
background: string background: string
@ -82,48 +81,36 @@ describe("/templates", () => {
}) })
describe("create app from template", () => { describe("create app from template", () => {
it.each(["sqs", "lucene"])( it("should be able to create an app from a template", async () => {
`should be able to create an app from a template (%s)`, const name = generator.guid().replaceAll("-", "")
async source => { const url = `/${name}`
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({ const app = await config.api.application.create({
name, name,
url, url,
useTemplate: "true", useTemplate: "true",
templateName: "Agency Client Portal", templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal", templateKey: "app/agency-client-portal",
}) })
expect(app.name).toBe(name) expect(app.name).toBe(name)
expect(app.url).toBe(url) expect(app.url).toBe(url)
await config.withApp(app, async () => { await config.withApp(app, async () => {
const tables = await config.api.table.fetch() const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2) expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name)) tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects") expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users") expect(users.name).toBe("Users")
const { rows } = await config.api.row.search( const { rows } = await config.api.row.search(agencyProjects._id!, {
agencyProjects._id!, tableId: agencyProjects._id!,
{ query: {},
tableId: agencyProjects._id!, })
query: {},
}
)
expect(rows).toHaveLength(3) expect(rows).toHaveLength(3)
}) })
} })
)
}
)
}) })
}) })

7279
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

File diff suppressed because it is too large Load Diff

79
packages/server/src/api/routes/utils/validators.ts
View File

@ -9,6 +9,13 @@ import {
Table, Table,
WebhookActionType, WebhookActionType,
BuiltinPermissionID, BuiltinPermissionID,
ViewV2Type,
SortOrder,
SortType,
UILogicalOperator,
BasicOperator,
ArrayOperator,
RangeOperator,
} from "@budibase/types" } from "@budibase/types"
import Joi, { CustomValidator } from "joi" import Joi, { CustomValidator } from "joi"
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core" import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
@ -66,6 +73,66 @@ export function tableValidator() {
) )
} }
function searchUIFilterValidator() {
const logicalOperator = Joi.string().valid(
...Object.values(UILogicalOperator)
)
const operators = [
...Object.values(BasicOperator),
...Object.values(ArrayOperator),
...Object.values(RangeOperator),
]
const filters = Joi.array().items(
Joi.object({
operator: Joi.string()
.valid(...operators)
.required(),
field: Joi.string().required(),
// could do with better validation of value based on operator
value: Joi.any().required(),
})
)
return Joi.object({
logicalOperator,
onEmptyFilter: Joi.string().valid(...Object.values(EmptyFilterOption)),
groups: Joi.array().items(
Joi.object({
logicalOperator,
filters,
groups: Joi.array().items(
Joi.object({
filters,
logicalOperator,
})
),
})
),
})
}
export function viewValidator() {
return auth.joiValidator.body(
Joi.object({
id: OPTIONAL_STRING,
tableId: Joi.string().required(),
name: Joi.string().required(),
type: Joi.string().optional().valid(null, ViewV2Type.CALCULATION),
primaryDisplay: OPTIONAL_STRING,
schema: Joi.object().required(),
query: searchUIFilterValidator().optional(),
sort: Joi.object({
field: Joi.string().required(),
order: Joi.string()
.optional()
.valid(...Object.values(SortOrder)),
type: Joi.string()
.optional()
.valid(...Object.values(SortType)),
}).optional(),
})
)
}
export function nameValidator() { export function nameValidator() {
return auth.joiValidator.body( return auth.joiValidator.body(
Joi.object({ Joi.object({
@ -91,8 +158,7 @@ export function datasourceValidator() {
) )
} }
function filterObject(opts?: { unknown: boolean }) { function searchFiltersValidator() {
const { unknown = true } = opts || {}
const conditionalFilteringObject = () => const conditionalFilteringObject = () =>
Joi.object({ Joi.object({
conditions: Joi.array().items(Joi.link("#schema")).required(), conditions: Joi.array().items(Joi.link("#schema")).required(),
@ -119,7 +185,14 @@ function filterObject(opts?: { unknown: boolean }) {
fuzzyOr: Joi.forbidden(), fuzzyOr: Joi.forbidden(),
documentType: Joi.forbidden(), documentType: Joi.forbidden(),
} }
return Joi.object(filtersValidators).unknown(unknown).id("schema")
return Joi.object(filtersValidators)
}
function filterObject(opts?: { unknown: boolean }) {
const { unknown = true } = opts || {}
return searchFiltersValidator().unknown(unknown).id("schema")
} }
export function internalSearchValidator() { export function internalSearchValidator() {

5
packages/server/src/api/routes/view.ts
View File

@ -8,6 +8,11 @@ import { permissions } from "@budibase/backend-core"
const router: Router = new Router() const router: Router = new Router()
router router
.get(
"/api/v2/views",
authorized(permissions.BUILDER),
viewController.v2.fetch
)
.get( .get(
"/api/v2/views/:viewId", "/api/v2/views/:viewId",
authorizedResource( authorizedResource(

89
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -1,10 +1,6 @@
import * as setup from "../../../api/routes/tests/utilities" import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures" import { basicTable } from "../../../tests/utilities/structures"
import { import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
} from "@budibase/backend-core"
import { import {
LinkDocument, LinkDocument,
DocumentType, DocumentType,
@ -70,24 +66,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
} }
} }
async function sqsDisabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => { describe("SQS migration", () => {
beforeAll(async () => { beforeAll(async () => {
await sqsDisabled(async () => { await config.init()
await config.init() const table = await config.api.table.save(basicTable())
const table = await config.api.table.save(basicTable()) tableId = table._id!
tableId = table._id! const db = dbCore.getDB(config.appId!)
const db = dbCore.getDB(config.appId!) // old link document
// old link document await db.put(oldLinkDocument())
await db.put(oldLinkDocument())
})
}) })
beforeEach(async () => { beforeEach(async () => {
@ -101,43 +87,32 @@ describe("SQS migration", () => {
it("test migration runs as expected against an older DB", async () => { it("test migration runs as expected against an older DB", async () => {
const db = dbCore.getDB(config.appId!) const db = dbCore.getDB(config.appId!)
// confirm nothing exists initially
await sqsDisabled(async () => { // remove sqlite design doc to simulate it comes from an older installation
let error: any | undefined const doc = await db.get(SQLITE_DESIGN_DOC_ID)
try { await db.remove({ _id: doc._id, _rev: doc._rev })
await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err: any) { await processMigrations(config.appId!, MIGRATIONS)
error = err const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
} expect(designDoc.sql.tables).toBeDefined()
expect(error).toBeDefined() const mainTableDef = designDoc.sql.tables[tableId]
expect(error.status).toBe(404) expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
}) })
await sqsEnabled(async () => { const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
await processMigrations(config.appId!, MIGRATIONS) const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID) expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2))
expect(designDoc.sql.tables).toBeDefined() // should have swapped the documents
const mainTableDef = designDoc.sql.tables[tableId] expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(mainTableDef).toBeDefined() expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(mainTableDef.fields[prefix("name")]).toEqual({ expect(linkDoc.doc2.tableId).toEqual(tableId1)
field: "name", expect(linkDoc.doc2.rowId).toEqual(rowId1)
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(
generateJunctionTableID(tableId1, tableId2)
)
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
}) })
}) })

10
packages/server/src/automations/tests/bash.spec.js → packages/server/src/automations/tests/bash.spec.ts
View File

@ -1,15 +1,15 @@
const setup = require("./utilities") import { getConfig, afterAll as _afterAll, runStep } from "./utilities"
describe("test the bash action", () => { describe("test the bash action", () => {
let config = setup.getConfig() let config = getConfig()
beforeAll(async () => { beforeAll(async () => {
await config.init() await config.init()
}) })
afterAll(setup.afterAll) afterAll(_afterAll)
it("should be able to execute a script", async () => { it("should be able to execute a script", async () => {
let res = await setup.runStep("EXECUTE_BASH", { let res = await runStep(config, "EXECUTE_BASH", {
code: "echo 'test'", code: "echo 'test'",
}) })
expect(res.stdout).toEqual("test\n") expect(res.stdout).toEqual("test\n")
@ -17,7 +17,7 @@ describe("test the bash action", () => {
}) })
it("should handle a null value", async () => { it("should handle a null value", async () => {
let res = await setup.runStep("EXECUTE_BASH", { let res = await runStep(config, "EXECUTE_BASH", {
code: null, code: null,
}) })
expect(res.stdout).toEqual( expect(res.stdout).toEqual(

12
packages/server/src/automations/tests/createRow.spec.ts
View File

@ -31,7 +31,7 @@ describe("test the create row action", () => {
afterAll(setup.afterAll) afterAll(setup.afterAll)
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row, row,
}) })
expect(res.id).toBeDefined() expect(res.id).toBeDefined()
@ -43,7 +43,7 @@ describe("test the create row action", () => {
}) })
it("should return an error (not throw) when bad info provided", async () => { it("should return an error (not throw) when bad info provided", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: { row: {
tableId: "invalid", tableId: "invalid",
invalid: "invalid", invalid: "invalid",
@ -53,7 +53,7 @@ describe("test the create row action", () => {
}) })
it("should check invalid inputs return an error", async () => { it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {}) const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {})
expect(res.success).toEqual(false) expect(res.success).toEqual(false)
}) })
@ -76,7 +76,7 @@ describe("test the create row action", () => {
] ]
attachmentRow.file_attachment = attachmentObject attachmentRow.file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow, row: attachmentRow,
}) })
@ -111,7 +111,7 @@ describe("test the create row action", () => {
} }
attachmentRow.single_file_attachment = attachmentObject attachmentRow.single_file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow, row: attachmentRow,
}) })
@ -146,7 +146,7 @@ describe("test the create row action", () => {
} }
attachmentRow.single_file_attachment = attachmentObject attachmentRow.single_file_attachment = attachmentObject
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.CREATE_ROW.stepId, {
row: attachmentRow, row: attachmentRow,
}) })

14
packages/server/src/automations/tests/delay.spec.js → packages/server/src/automations/tests/delay.spec.ts
View File

@ -1,14 +1,20 @@
const setup = require("./utilities") import { runStep, actions, getConfig } from "./utilities"
import { reset } from "timekeeper"
// need real Date for this test // need real Date for this test
const tk = require("timekeeper") reset()
tk.reset()
describe("test the delay logic", () => { describe("test the delay logic", () => {
const config = getConfig()
beforeAll(async () => {
await config.init()
})
it("should be able to run the delay", async () => { it("should be able to run the delay", async () => {
const time = 100 const time = 100
const before = Date.now() const before = Date.now()
await setup.runStep(setup.actions.DELAY.stepId, { time: time }) await runStep(config, actions.DELAY.stepId, { time: time })
const now = Date.now() const now = Date.now()
// divide by two just so that test will always pass as long as there was some sort of delay // divide by two just so that test will always pass as long as there was some sort of delay
expect(now - before).toBeGreaterThanOrEqual(time / 2) expect(now - before).toBeGreaterThanOrEqual(time / 2)

21
packages/server/src/automations/tests/deleteRow.spec.ts
View File

@ -1,4 +1,4 @@
const setup = require("./utilities") import * as setup from "./utilities"
describe("test the delete row action", () => { describe("test the delete row action", () => {
let table: any let table: any
@ -20,32 +20,29 @@ describe("test the delete row action", () => {
afterAll(setup.afterAll) afterAll(setup.afterAll)
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.DELETE_ROW.stepId,
inputs
)
expect(res.success).toEqual(true) expect(res.success).toEqual(true)
expect(res.response).toBeDefined() expect(res.response).toBeDefined()
expect(res.row._id).toEqual(row._id) expect(res.row._id).toEqual(row._id)
let error
try {
await config.getRow(table._id, res.row._id)
} catch (err) {
error = err
}
expect(error).toBeDefined()
}) })
it("check usage quota attempts", async () => { it("check usage quota attempts", async () => {
await setup.runInProd(async () => { await setup.runInProd(async () => {
await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs) await setup.runStep(config, setup.actions.DELETE_ROW.stepId, inputs)
}) })
}) })
it("should check invalid inputs return an error", async () => { it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, {}) const res = await setup.runStep(config, setup.actions.DELETE_ROW.stepId, {})
expect(res.success).toEqual(false) expect(res.success).toEqual(false)
}) })
it("should return an error when table doesn't exist", async () => { it("should return an error when table doesn't exist", async () => {
const res = await setup.runStep(setup.actions.DELETE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.DELETE_ROW.stepId, {
tableId: "invalid", tableId: "invalid",
id: "invalid", id: "invalid",
revision: "invalid", revision: "invalid",

2
packages/server/src/automations/tests/discord.spec.ts
View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" }) nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.discord.stepId, { const res = await runStep(config, actions.discord.stepId, {
url: "http://www.example.com", url: "http://www.example.com",
username: "joe_bloggs", username: "joe_bloggs",
}) })

135
packages/server/src/automations/tests/executeQuery.spec.ts
View File

@ -1,65 +1,80 @@
import { Datasource, Query } from "@budibase/types" import { Datasource, Query } from "@budibase/types"
import * as setup from "./utilities" import * as setup from "./utilities"
import { DatabaseName } from "../../integrations/tests/utils" import {
DatabaseName,
datasourceDescribe,
} from "../../integrations/tests/utils"
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
describe.each([ const descriptions = datasourceDescribe({
DatabaseName.POSTGRES, exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
DatabaseName.MYSQL,
DatabaseName.SQL_SERVER,
DatabaseName.MARIADB,
DatabaseName.ORACLE,
])("execute query action (%s)", name => {
let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
const config = setup.getConfig()
beforeAll(async () => {
await config.init()
const testSetup = await setup.setupTestDatasource(config, name)
datasource = testSetup.datasource
client = testSetup.client
})
beforeEach(async () => {
tableName = await setup.createTestTable(client, {
a: { type: "string" },
b: { type: "number" },
})
await setup.insertTestData(client, tableName, [{ a: "string", b: 1 }])
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
afterEach(async () => {
await client.schema.dropTable(tableName)
})
afterAll(setup.afterAll)
it("should be able to execute a query", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: { queryId: query._id },
})
expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(true)
})
it("should handle a null query value", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: null,
})
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false)
})
it("should handle an error executing a query", async () => {
let res = await setup.runStep(setup.actions.EXECUTE_QUERY.stepId, {
query: { queryId: "wrong_id" },
})
expect(res.response).toBeDefined()
expect(res.success).toEqual(false)
})
}) })
if (descriptions.length) {
describe.each(descriptions)(
"execute query action ($dbName)",
({ config, dsProvider }) => {
let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
beforeEach(async () => {
tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.string("a")
table.integer("b")
})
await client(tableName).insert({ a: "string", b: 1 })
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
afterEach(async () => {
await client.schema.dropTable(tableName)
})
it("should be able to execute a query", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: query._id },
}
)
expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(true)
})
it("should handle a null query value", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: null,
}
)
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false)
})
it("should handle an error executing a query", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: "wrong_id" },
}
)
expect(res.response).toBeDefined()
expect(res.success).toEqual(false)
})
}
)
}

17
packages/server/src/automations/tests/executeScript.spec.js → packages/server/src/automations/tests/executeScript.spec.ts
View File

@ -1,15 +1,15 @@
const setup = require("./utilities") import { getConfig, afterAll as _afterAll, runStep, actions } from "./utilities"
describe("test the execute script action", () => { describe("test the execute script action", () => {
let config = setup.getConfig() let config = getConfig()
beforeAll(async () => { beforeAll(async () => {
await config.init() await config.init()
}) })
afterAll(setup.afterAll) afterAll(_afterAll)
it("should be able to execute a script", async () => { it("should be able to execute a script", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, { const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: "return 1 + 1", code: "return 1 + 1",
}) })
expect(res.value).toEqual(2) expect(res.value).toEqual(2)
@ -17,7 +17,7 @@ describe("test the execute script action", () => {
}) })
it("should handle a null value", async () => { it("should handle a null value", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, { const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: null, code: null,
}) })
expect(res.response.message).toEqual("Invalid inputs") expect(res.response.message).toEqual("Invalid inputs")
@ -25,8 +25,9 @@ describe("test the execute script action", () => {
}) })
it("should be able to get a value from context", async () => { it("should be able to get a value from context", async () => {
const res = await setup.runStep( const res = await runStep(
setup.actions.EXECUTE_SCRIPT.stepId, config,
actions.EXECUTE_SCRIPT.stepId,
{ {
code: "return steps.map(d => d.value)", code: "return steps.map(d => d.value)",
}, },
@ -40,7 +41,7 @@ describe("test the execute script action", () => {
}) })
it("should be able to handle an error gracefully", async () => { it("should be able to handle an error gracefully", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, { const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: "return something.map(x => x.name)", code: "return something.map(x => x.name)",
}) })
expect(res.response).toEqual("ReferenceError: something is not defined") expect(res.response).toEqual("ReferenceError: something is not defined")

8
packages/server/src/automations/tests/filter.spec.ts
View File

@ -2,13 +2,19 @@ import * as setup from "./utilities"
import { FilterConditions } from "../steps/filter" import { FilterConditions } from "../steps/filter"
describe("test the filter logic", () => { describe("test the filter logic", () => {
const config = setup.getConfig()
beforeAll(async () => {
await config.init()
})
async function checkFilter( async function checkFilter(
field: any, field: any,
condition: string, condition: string,
value: any, value: any,
pass = true pass = true
) { ) {
let res = await setup.runStep(setup.actions.FILTER.stepId, { let res = await setup.runStep(config, setup.actions.FILTER.stepId, {
field, field,
condition, condition,
value, value,

6
packages/server/src/automations/tests/make.spec.ts
View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" }) nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.integromat.stepId, { const res = await runStep(config, actions.integromat.stepId, {
url: "http://www.example.com", url: "http://www.example.com",
}) })
expect(res.response.foo).toEqual("bar") expect(res.response.foo).toEqual("bar")
@ -38,7 +38,7 @@ describe("test the outgoing webhook action", () => {
.post("/", payload) .post("/", payload)
.reply(200, { foo: "bar" }) .reply(200, { foo: "bar" })
const res = await runStep(actions.integromat.stepId, { const res = await runStep(config, actions.integromat.stepId, {
body: { value: JSON.stringify(payload) }, body: { value: JSON.stringify(payload) },
url: "http://www.example.com", url: "http://www.example.com",
}) })
@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
}) })
it("should return a 400 if the JSON payload string is malformed", async () => { it("should return a 400 if the JSON payload string is malformed", async () => {
const res = await runStep(actions.integromat.stepId, { const res = await runStep(config, actions.integromat.stepId, {
body: { value: "{ invalid json }" }, body: { value: "{ invalid json }" },
url: "http://www.example.com", url: "http://www.example.com",
}) })

8
packages/server/src/automations/tests/n8n.spec.ts
View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action and default to 'get'", async () => { it("should be able to run the action and default to 'get'", async () => {
nock("http://www.example.com/").get("/").reply(200, { foo: "bar" }) nock("http://www.example.com/").get("/").reply(200, { foo: "bar" })
const res = await runStep(actions.n8n.stepId, { const res = await runStep(config, actions.n8n.stepId, {
url: "http://www.example.com", url: "http://www.example.com",
body: { body: {
test: "IGNORE_ME", test: "IGNORE_ME",
@ -30,7 +30,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com/") nock("http://www.example.com/")
.post("/", { name: "Adam", age: 9 }) .post("/", { name: "Adam", age: 9 })
.reply(200) .reply(200)
const res = await runStep(actions.n8n.stepId, { const res = await runStep(config, actions.n8n.stepId, {
body: { body: {
value: JSON.stringify({ name: "Adam", age: 9 }), value: JSON.stringify({ name: "Adam", age: 9 }),
}, },
@ -42,7 +42,7 @@ describe("test the outgoing webhook action", () => {
it("should return a 400 if the JSON payload string is malformed", async () => { it("should return a 400 if the JSON payload string is malformed", async () => {
const payload = `{ value1 1 }` const payload = `{ value1 1 }`
const res = await runStep(actions.n8n.stepId, { const res = await runStep(config, actions.n8n.stepId, {
value1: "ONE", value1: "ONE",
body: { body: {
value: payload, value: payload,
@ -59,7 +59,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com/") nock("http://www.example.com/")
.head("/", body => body === "") .head("/", body => body === "")
.reply(200) .reply(200)
const res = await runStep(actions.n8n.stepId, { const res = await runStep(config, actions.n8n.stepId, {
url: "http://www.example.com", url: "http://www.example.com",
method: "HEAD", method: "HEAD",
body: { body: {

8
packages/server/src/automations/tests/openai.spec.ts
View File

@ -62,13 +62,13 @@ describe("test the openai action", () => {
afterAll(_afterAll) afterAll(_afterAll)
it("should be able to receive a response from ChatGPT given a prompt", async () => { it("should be able to receive a response from ChatGPT given a prompt", async () => {
const res = await runStep("OPENAI", { prompt: OPENAI_PROMPT }) const res = await runStep(config, "OPENAI", { prompt: OPENAI_PROMPT })
expect(res.response).toEqual("This is a test") expect(res.response).toEqual("This is a test")
expect(res.success).toBeTruthy() expect(res.success).toBeTruthy()
}) })
it("should present the correct error message when a prompt is not provided", async () => { it("should present the correct error message when a prompt is not provided", async () => {
const res = await runStep("OPENAI", { prompt: null }) const res = await runStep(config, "OPENAI", { prompt: null })
expect(res.response).toEqual( expect(res.response).toEqual(
"Budibase OpenAI Automation Failed: No prompt supplied" "Budibase OpenAI Automation Failed: No prompt supplied"
) )
@ -91,7 +91,7 @@ describe("test the openai action", () => {
} as any) } as any)
) )
const res = await runStep("OPENAI", { const res = await runStep(config, "OPENAI", {
prompt: OPENAI_PROMPT, prompt: OPENAI_PROMPT,
}) })
@ -106,7 +106,7 @@ describe("test the openai action", () => {
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?" const prompt = "What is the meaning of life?"
await runStep("OPENAI", { await runStep(config, "OPENAI", {
model: "gpt-4o-mini", model: "gpt-4o-mini",
prompt, prompt,
}) })

4
packages/server/src/automations/tests/outgoingWebhook.spec.ts
View File

@ -18,7 +18,7 @@ describe("test the outgoing webhook action", () => {
nock("http://www.example.com") nock("http://www.example.com")
.post("/", { a: 1 }) .post("/", { a: 1 })
.reply(200, { foo: "bar" }) .reply(200, { foo: "bar" })
const res = await runStep(actions.OUTGOING_WEBHOOK.stepId, { const res = await runStep(config, actions.OUTGOING_WEBHOOK.stepId, {
requestMethod: "POST", requestMethod: "POST",
url: "www.example.com", url: "www.example.com",
requestBody: JSON.stringify({ a: 1 }), requestBody: JSON.stringify({ a: 1 }),
@ -28,7 +28,7 @@ describe("test the outgoing webhook action", () => {
}) })
it("should return an error if something goes wrong in fetch", async () => { it("should return an error if something goes wrong in fetch", async () => {
const res = await runStep(actions.OUTGOING_WEBHOOK.stepId, { const res = await runStep(config, actions.OUTGOING_WEBHOOK.stepId, {
requestMethod: "GET", requestMethod: "GET",
url: "www.invalid.com", url: "www.invalid.com",
}) })

30
packages/server/src/automations/tests/queryRows.spec.ts
View File

@ -33,7 +33,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending", sortOrder: "ascending",
limit: 10, limit: 10,
} }
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true) expect(res.success).toBe(true)
expect(res.rows).toBeDefined() expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2) expect(res.rows.length).toBe(2)
@ -48,7 +52,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending", sortOrder: "ascending",
limit: 10, limit: 10,
} }
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true) expect(res.success).toBe(true)
expect(res.rows).toBeDefined() expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2) expect(res.rows.length).toBe(2)
@ -65,7 +73,11 @@ describe("Test a query step automation", () => {
limit: 10, limit: 10,
onEmptyFilter: "none", onEmptyFilter: "none",
} }
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(false) expect(res.success).toBe(false)
expect(res.rows).toBeDefined() expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0) expect(res.rows.length).toBe(0)
@ -85,7 +97,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending", sortOrder: "ascending",
limit: 10, limit: 10,
} }
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(false) expect(res.success).toBe(false)
expect(res.rows).toBeDefined() expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(0) expect(res.rows.length).toBe(0)
@ -100,7 +116,11 @@ describe("Test a query step automation", () => {
sortOrder: "ascending", sortOrder: "ascending",
limit: 10, limit: 10,
} }
const res = await setup.runStep(setup.actions.QUERY_ROWS.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.QUERY_ROWS.stepId,
inputs
)
expect(res.success).toBe(true) expect(res.success).toBe(true)
expect(res.rows).toBeDefined() expect(res.rows).toBeDefined()
expect(res.rows.length).toBe(2) expect(res.rows.length).toBe(2)

201
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -1,9 +1,14 @@
import * as automation from "../../index" import * as automation from "../../index"
import * as setup from "../utilities" import * as setup from "../utilities"
import { LoopStepType, FieldType, Table } from "@budibase/types" import { LoopStepType, FieldType, Table, Datasource } from "@budibase/types"
import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
import { DatabaseName } from "../../../integrations/tests/utils" import {
DatabaseName,
datasourceDescribe,
} from "../../../integrations/tests/utils"
import { FilterConditions } from "../../../automations/steps/filter" import { FilterConditions } from "../../../automations/steps/filter"
import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
describe("Automation Scenarios", () => { describe("Automation Scenarios", () => {
let config = setup.getConfig() let config = setup.getConfig()
@ -107,96 +112,6 @@ describe("Automation Scenarios", () => {
expect(results.steps[2].outputs.rows).toHaveLength(1) expect(results.steps[2].outputs.rows).toHaveLength(1)
}) })
it("should query an external database for some data then insert than into an internal table", async () => {
const { datasource, client } = await setup.setupTestDatasource(
config,
DatabaseName.MYSQL
)
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = await setup.createTestTable(client, {
name: { type: "string" },
age: { type: "number" },
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await setup.insertTestData(client, tableName, rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
)
})
})
it("should trigger an automation which creates and then updates a row", async () => { it("should trigger an automation which creates and then updates a row", async () => {
const table = await config.createTable({ const table = await config.createTable({
name: "TestTable", name: "TestTable",
@ -517,3 +432,105 @@ describe("Automation Scenarios", () => {
expect(results.steps[0].outputs.message).toContain("example.com") expect(results.steps[0].outputs.message).toContain("example.com")
}) })
}) })
const descriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
if (descriptions.length) {
describe.each(descriptions)("/rows ($dbName)", ({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
beforeAll(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
client = ds.client!
})
it("should query an external database for some data then insert than into an internal table", async () => {
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
table.string("name")
table.integer("age")
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await client(tableName).insert(rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
config,
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
)
})
})
})
}

3
packages/server/src/automations/tests/sendSmtpEmail.spec.ts
View File

@ -18,7 +18,7 @@ function generateResponse(to: string, from: string) {
} }
} }
const setup = require("./utilities") import * as setup from "./utilities"
describe("test the outgoing webhook action", () => { describe("test the outgoing webhook action", () => {
let inputs let inputs
@ -58,6 +58,7 @@ describe("test the outgoing webhook action", () => {
} }
let resp = generateResponse(inputs.to, inputs.from) let resp = generateResponse(inputs.to, inputs.from)
const res = await setup.runStep( const res = await setup.runStep(
config,
setup.actions.SEND_EMAIL_SMTP.stepId, setup.actions.SEND_EMAIL_SMTP.stepId,
inputs inputs
) )

10
packages/server/src/automations/tests/serverLog.spec.js → packages/server/src/automations/tests/serverLog.spec.ts
View File

@ -1,8 +1,8 @@
const setup = require("./utilities") import { getConfig, afterAll as _afterAll, runStep, actions } from "./utilities"
describe("test the server log action", () => { describe("test the server log action", () => {
let config = setup.getConfig() let config = getConfig()
let inputs let inputs: any
beforeAll(async () => { beforeAll(async () => {
await config.init() await config.init()
@ -10,10 +10,10 @@ describe("test the server log action", () => {
text: "log message", text: "log message",
} }
}) })
afterAll(setup.afterAll) afterAll(_afterAll)
it("should be able to log the text", async () => { it("should be able to log the text", async () => {
let res = await setup.runStep(setup.actions.SERVER_LOG.stepId, inputs) let res = await runStep(config, actions.SERVER_LOG.stepId, inputs)
expect(res.message).toEqual(`App ${config.getAppId()} - ${inputs.text}`) expect(res.message).toEqual(`App ${config.getAppId()} - ${inputs.text}`)
expect(res.success).toEqual(true) expect(res.success).toEqual(true)
}) })

2
packages/server/src/automations/tests/triggerAutomationRun.spec.ts
View File

@ -29,6 +29,7 @@ describe("Test triggering an automation from another automation", () => {
}, },
} }
const res = await setup.runStep( const res = await setup.runStep(
config,
setup.actions.TRIGGER_AUTOMATION_RUN.stepId, setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs inputs
) )
@ -44,6 +45,7 @@ describe("Test triggering an automation from another automation", () => {
}, },
} }
const res = await setup.runStep( const res = await setup.runStep(
config,
setup.actions.TRIGGER_AUTOMATION_RUN.stepId, setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs inputs
) )

68
packages/server/src/automations/tests/updateRow.spec.ts
View File

@ -34,7 +34,11 @@ describe("test the update row action", () => {
afterAll(setup.afterAll) afterAll(setup.afterAll)
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, inputs) const res = await setup.runStep(
config,
setup.actions.UPDATE_ROW.stepId,
inputs
)
expect(res.success).toEqual(true) expect(res.success).toEqual(true)
const updatedRow = await config.api.row.get(table._id!, res.id) const updatedRow = await config.api.row.get(table._id!, res.id)
expect(updatedRow.name).toEqual("Updated name") expect(updatedRow.name).toEqual("Updated name")
@ -42,12 +46,12 @@ describe("test the update row action", () => {
}) })
it("should check invalid inputs return an error", async () => { it("should check invalid inputs return an error", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, {}) const res = await setup.runStep(config, setup.actions.UPDATE_ROW.stepId, {})
expect(res.success).toEqual(false) expect(res.success).toEqual(false)
}) })
it("should return an error when table doesn't exist", async () => { it("should return an error when table doesn't exist", async () => {
const res = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { const res = await setup.runStep(config, setup.actions.UPDATE_ROW.stepId, {
row: { _id: "invalid" }, row: { _id: "invalid" },
rowId: "invalid", rowId: "invalid",
}) })
@ -90,16 +94,20 @@ describe("test the update row action", () => {
expect(getResp.user1[0]._id).toEqual(user1._id) expect(getResp.user1[0]._id).toEqual(user1._id)
expect(getResp.user2[0]._id).toEqual(user2._id) expect(getResp.user2[0]._id).toEqual(user2._id)
let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { let stepResp = await setup.runStep(
rowId: row._id, config,
row: { setup.actions.UPDATE_ROW.stepId,
_id: row._id, {
_rev: row._rev, rowId: row._id,
tableId: row.tableId, row: {
user1: [user2._id], _id: row._id,
user2: "", _rev: row._rev,
}, tableId: row.tableId,
}) user1: [user2._id],
user2: "",
},
}
)
expect(stepResp.success).toEqual(true) expect(stepResp.success).toEqual(true)
getResp = await config.api.row.get(table._id!, row._id!) getResp = await config.api.row.get(table._id!, row._id!)
@ -143,23 +151,27 @@ describe("test the update row action", () => {
expect(getResp.user1[0]._id).toEqual(user1._id) expect(getResp.user1[0]._id).toEqual(user1._id)
expect(getResp.user2[0]._id).toEqual(user2._id) expect(getResp.user2[0]._id).toEqual(user2._id)
let stepResp = await setup.runStep(setup.actions.UPDATE_ROW.stepId, { let stepResp = await setup.runStep(
rowId: row._id, config,
row: { setup.actions.UPDATE_ROW.stepId,
_id: row._id, {
_rev: row._rev, rowId: row._id,
tableId: row.tableId, row: {
user1: [user2._id], _id: row._id,
user2: "", _rev: row._rev,
}, tableId: row.tableId,
meta: { user1: [user2._id],
fields: { user2: "",
user2: { },
clearRelationships: true, meta: {
fields: {
user2: {
clearRelationships: true,
},
}, },
}, },
}, }
}) )
expect(stepResp.success).toEqual(true) expect(stepResp.success).toEqual(true)
getResp = await config.api.row.get(table._id!, row._id!) getResp = await config.api.row.get(table._id!, row._id!)

58
packages/server/src/automations/tests/utilities/index.ts
View File

@ -1,22 +1,16 @@
import TestConfig from "../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions" import { BUILTIN_ACTION_DEFINITIONS, getAction } from "../../actions"
import emitter from "../../../events/index" import emitter from "../../../events/index"
import env from "../../../environment" import env from "../../../environment"
import { AutomationActionStepId, Datasource } from "@budibase/types" import { AutomationActionStepId, Datasource } from "@budibase/types"
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests"
import {
getDatasource,
knexClient,
DatabaseName,
} from "../../../integrations/tests/utils"
let config: TestConfig let config: TestConfiguration
export function getConfig(): TestConfig { export function getConfig(): TestConfiguration {
if (!config) { if (!config) {
config = new TestConfig(true) config = new TestConfiguration(true)
} }
return config return config
} }
@ -39,7 +33,12 @@ export async function runInProd(fn: any) {
} }
} }
export async function runStep(stepId: string, inputs: any, stepContext?: any) { export async function runStep(
config: TestConfiguration,
stepId: string,
inputs: any,
stepContext?: any
) {
async function run() { async function run() {
let step = await getAction(stepId as AutomationActionStepId) let step = await getAction(stepId as AutomationActionStepId)
expect(step).toBeDefined() expect(step).toBeDefined()
@ -55,7 +54,7 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
emitter, emitter,
}) })
} }
if (config?.appId) { if (config.appId) {
return context.doInContext(config?.appId, async () => { return context.doInContext(config?.appId, async () => {
return run() return run()
}) })
@ -64,31 +63,8 @@ export async function runStep(stepId: string, inputs: any, stepContext?: any) {
} }
} }
export async function createTestTable(client: Knex, schema: any) {
const tableName = generator.guid()
await client.schema.createTable(tableName, table => {
for (const fieldName in schema) {
const field = schema[fieldName]
if (field.type === "string") {
table.string(fieldName)
} else if (field.type === "number") {
table.integer(fieldName)
}
}
})
return tableName
}
export async function insertTestData(
client: Knex,
tableName: string,
rows: any[]
) {
await client(tableName).insert(rows)
}
export async function saveTestQuery( export async function saveTestQuery(
config: TestConfig, config: TestConfiguration,
client: Knex, client: Knex,
tableName: string, tableName: string,
datasource: Datasource datasource: Datasource
@ -107,15 +83,5 @@ export async function saveTestQuery(
}) })
} }
export async function setupTestDatasource(
config: TestConfig,
dbName: DatabaseName
) {
const db = await getDatasource(dbName)
const datasource = await config.api.datasource.create(db)
const client = await knexClient(db)
return { datasource, client }
}
export const apiKey = "test" export const apiKey = "test"
export const actions = BUILTIN_ACTION_DEFINITIONS export const actions = BUILTIN_ACTION_DEFINITIONS

6
packages/server/src/automations/tests/zapier.spec.ts
View File

@ -16,7 +16,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => { it("should be able to run the action", async () => {
nock("http://www.example.com/").post("/").reply(200, { foo: "bar" }) nock("http://www.example.com/").post("/").reply(200, { foo: "bar" })
const res = await runStep(actions.zapier.stepId, { const res = await runStep(config, actions.zapier.stepId, {
url: "http://www.example.com", url: "http://www.example.com",
}) })
expect(res.response.foo).toEqual("bar") expect(res.response.foo).toEqual("bar")
@ -38,7 +38,7 @@ describe("test the outgoing webhook action", () => {
.post("/", { ...payload, platform: "budibase" }) .post("/", { ...payload, platform: "budibase" })
.reply(200, { foo: "bar" }) .reply(200, { foo: "bar" })
const res = await runStep(actions.zapier.stepId, { const res = await runStep(config, actions.zapier.stepId, {
body: { value: JSON.stringify(payload) }, body: { value: JSON.stringify(payload) },
url: "http://www.example.com", url: "http://www.example.com",
}) })
@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
}) })
it("should return a 400 if the JSON payload string is malformed", async () => { it("should return a 400 if the JSON payload string is malformed", async () => {
const res = await runStep(actions.zapier.stepId, { const res = await runStep(config, actions.zapier.stepId, {
body: { value: "{ invalid json }" }, body: { value: "{ invalid json }" },
url: "http://www.example.com", url: "http://www.example.com",
}) })

11
packages/server/src/db/linkedRows/index.ts
View File

@ -14,11 +14,10 @@ import {
coreOutputProcessing, coreOutputProcessing,
processFormulas, processFormulas,
} from "../../utilities/rowProcessor" } from "../../utilities/rowProcessor"
import { context, features } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { import {
ContextUser, ContextUser,
EventType, EventType,
FeatureFlag,
FieldType, FieldType,
LinkDocumentValue, LinkDocumentValue,
Row, Row,
@ -251,19 +250,13 @@ export async function squashLinks<T = Row[] | Row>(
source: Table | ViewV2, source: Table | ViewV2,
enriched: T enriched: T
): Promise<T> { ): Promise<T> {
const allowRelationshipSchemas = await features.flags.isEnabled(
FeatureFlag.ENRICHED_RELATIONSHIPS
)
let viewSchema: ViewV2Schema = {} let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) { if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) { if (helpers.views.isCalculationView(source)) {
return enriched return enriched
} }
if (allowRelationshipSchemas) { viewSchema = source.schema || {}
viewSchema = source.schema || {}
}
} }
let table: Table let table: Table

Some files were not shown because too many files have changed in this diff Show More