MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/update-automation-tests

This commit is contained in:
Peter Clement 2024-11-26 20:08:10 +00:00
parent 685657b8eb ef4da634a0
commit 977c0e44f3
114 changed files with 21637 additions and 18024 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintignore
View File

@ -9,8 +9,5 @@ packages/server/client
packages/server/coverage packages/server/coverage
packages/builder/.routify packages/builder/.routify
packages/sdk/sdk packages/sdk/sdk
packages/account-portal/packages/server/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/ui/build
**/*.ivm.bundle.js **/*.ivm.bundle.js
packages/server/build/oldClientVersions/**/** packages/server/build/oldClientVersions/**/**

3
.eslintrc.json
View File

@ -27,9 +27,8 @@
"extends": "plugin:svelte/recommended", "extends": "plugin:svelte/recommended",
"parser": "svelte-eslint-parser", "parser": "svelte-eslint-parser",
"parserOptions": { "parserOptions": {
"parser": "@babel/eslint-parser", "parser": "@typescript-eslint/parser",
"ecmaVersion": 2019, "ecmaVersion": 2019,
"sourceType": "module",
"allowImportExportEverywhere": true "allowImportExportEverywhere": true
} }
}, },

91
.github/workflows/budibase_ci.yml vendored
View File

@ -64,18 +64,15 @@ jobs:
- run: yarn --frozen-lockfile - run: yarn --frozen-lockfile
# Run build all the projects # Run build all the projects
- name: Build OSS - name: Build
run: yarn build:oss run: yarn build
- name: Build account portal
run: yarn build:account-portal
if: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }}
# Check the types of the projects built via esbuild # Check the types of the projects built via esbuild
- name: Check types - name: Check types
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn check:types --since=${{ env.NX_BASE_BRANCH }} --ignore @budibase/account-portal-server yarn check:types --since=${{ env.NX_BASE_BRANCH }}
else else
yarn check:types --ignore @budibase/account-portal-server yarn check:types
fi fi
helm-lint: helm-lint:
@ -117,9 +114,11 @@ jobs:
- name: Test - name: Test
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix --since=${{ env.NX_BASE_BRANCH }} -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --since=${{ env.NX_BASE_BRANCH }}
else else
yarn test --ignore=@budibase/worker --ignore=@budibase/server yarn test -- --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/builder --no-prefix -- --verbose --reporters=default --reporters=github-actions
yarn test -- --scope=@budibase/builder --no-prefix
fi fi
test-worker: test-worker:
@ -141,16 +140,22 @@ jobs:
- name: Test worker - name: Test worker
run: | run: |
if ${{ env.ONLY_AFFECTED_TASKS }}; then if ${{ env.ONLY_AFFECTED_TASKS }}; then
node scripts/run-affected.js --task=test --scope=@budibase/worker --since=${{ env.NX_BASE_BRANCH }} AFFECTED=$(yarn --silent nx show projects --affected -t test --base=${{ env.NX_BASE_BRANCH }} -p @budibase/worker)
else if [ -z "$AFFECTED" ]; then
yarn test --scope=@budibase/worker echo "No affected tests to run"
exit 0
fi
fi fi
cd packages/worker
yarn test --verbose --reporters=default --reporters=github-actions
test-server: test-server:
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
datasource: [mssql, mysql, postgres, mongodb, mariadb, oracle, none] datasource:
[mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
steps: steps:
- name: Checkout repo - name: Checkout repo
uses: actions/checkout@v4 uses: actions/checkout@v4
@ -213,7 +218,7 @@ jobs:
fi fi
cd packages/server cd packages/server
yarn test --filter $FILTER --passWithNoTests yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
check-pro-submodule: check-pro-submodule:
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -274,64 +279,6 @@ jobs:
echo 'All good, the submodule had been merged and setup correctly!' echo 'All good, the submodule had been merged and setup correctly!'
fi fi
check-accountportal-submodule:
runs-on: ubuntu-latest
if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v4
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
fetch-depth: 0
- uses: dorny/paths-filter@v3
id: changes
with:
filters: |
src:
- packages/account-portal/**
- if: steps.changes.outputs.src == 'true'
name: Check account portal commit
id: get_accountportal_commits
run: |
cd packages/account-portal
accountportal_commit=$(git rev-parse HEAD)
branch="${{ github.base_ref || github.ref_name }}"
echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})"
base_commit=$(git rev-parse origin/master)
if [[ ! -z $base_commit ]]; then
echo "target_branch=$branch"
echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
echo "accountportal_commit=$accountportal_commit"
echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT"
echo "base_commit=$base_commit"
echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
else
echo "Nothing to do - branch to branch merge."
fi
- name: Check submodule merged to base branch
if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }}
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}';
const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}';
if (submoduleCommit !== baseCommit) {
console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.');
console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md')
process.exit(1);
} else {
console.log('All good, the submodule had been merged and setup correctly!')
}
check-lockfile: check-lockfile:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:

3
.gitignore vendored
View File

@ -8,6 +8,7 @@ packages/server/build/oldClientVersions/**/*
packages/builder/src/components/deploy/clientVersions.json packages/builder/src/components/deploy/clientVersions.json
packages/server/src/integrations/tests/utils/*.lock packages/server/src/integrations/tests/utils/*.lock
packages/builder/vite.config.mjs.timestamp* packages/builder/vite.config.mjs.timestamp*
packages/account-portal
# Logs # Logs
logs logs
@ -110,4 +111,4 @@ budibase-component
budibase-datasource budibase-datasource
*.iml *.iml
.nx .nx

3
.gitmodules vendored
View File

@ -1,6 +1,3 @@
[submodule "packages/pro"] [submodule "packages/pro"]
path = packages/pro path = packages/pro
url = git@github.com:Budibase/budibase-pro.git url = git@github.com:Budibase/budibase-pro.git
[submodule "packages/account-portal"]
path = packages/account-portal
url = git@github.com:Budibase/account-portal.git

4
.prettierignore
View File

@ -9,8 +9,4 @@ packages/backend-core/coverage
packages/builder/.routify packages/builder/.routify
packages/sdk/sdk packages/sdk/sdk
packages/pro/coverage packages/pro/coverage
packages/account-portal/packages/ui/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/server/build
packages/account-portal/packages/server/coverage
**/*.ivm.bundle.js **/*.ivm.bundle.js

10
.vscode/launch.json vendored
View File

@ -20,16 +20,6 @@
"args": ["${workspaceFolder}/packages/worker/src/index.ts"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"],
"cwd": "${workspaceFolder}/packages/worker" "cwd": "${workspaceFolder}/packages/worker"
}, },
{
"name": "Camunda Worker",
"type": "node",
"request": "launch",
"runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"],
"args": [
"${workspaceFolder}/packages/account-portal/packages/server/src/v2/run.ts"
],
"cwd": "${workspaceFolder}/packages/account-portal/packages/server"
},
{ {
"type": "chrome", "type": "chrome",
"request": "launch", "request": "launch",

12
charts/budibase/templates/secrets.yaml
View File

@ -12,12 +12,12 @@ metadata:
type: Opaque type: Opaque
data: data:
{{- if $existingSecret }} {{- if $existingSecret }}
internalApiKey: {{ index $existingSecret.data "internalApiKey" }} internalApiKey: {{ index $existingSecret.data "internalApiKey" | quote }}
jwtSecret: {{ index $existingSecret.data "jwtSecret" }} jwtSecret: {{ index $existingSecret.data "jwtSecret" | quote }}
objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" }} objectStoreAccess: {{ index $existingSecret.data "objectStoreAccess" | quote }}
objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" }} objectStoreSecret: {{ index $existingSecret.data "objectStoreSecret" | quote }}
bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" }} bbEncryptionKey: {{ index $existingSecret.data "bbEncryptionKey" | quote }}
apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" }} apiEncryptionKey: {{ index $existingSecret.data "apiEncryptionKey" | quote }}
{{- else }} {{- else }}
internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }} internalApiKey: {{ template "budibase.defaultsecret" .Values.globals.internalApiKey }}
jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }} jwtSecret: {{ template "budibase.defaultsecret" .Values.globals.jwtSecret }}

6
examples/nextjs-api-sales/yarn.lock
View File

@ -423,9 +423,9 @@ core-js-pure@^3.20.2:
integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ== integrity sha512-12VZfFIu+wyVbBebyHmRTuEE/tZrB4tJToWcwAMcsp3h4+sHR+fMJWbKpYiCRWlhFBq+KNyO8rIV9rTkeVmznQ==
cross-spawn@^7.0.2: cross-spawn@^7.0.2:
version "7.0.3" version "7.0.6"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f"
integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==
dependencies: dependencies:
path-key "^3.1.0" path-key "^3.1.0"
shebang-command "^2.0.0" shebang-command "^2.0.0"

7
lerna.json
View File

@ -1,12 +1,7 @@
{ {
"$schema": "node_modules/lerna/schemas/lerna-schema.json", "$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.5", "version": "3.2.12",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [
"packages/*",
"!packages/account-portal",
"packages/account-portal/packages/*"
],
"concurrency": 20, "concurrency": 20,
"command": { "command": {
"publish": { "publish": {

1
nx.json
View File

@ -2,7 +2,6 @@
"$schema": "./node_modules/nx/schemas/nx-schema.json", "$schema": "./node_modules/nx/schemas/nx-schema.json",
"tasksRunnerOptions": { "tasksRunnerOptions": {
"default": { "default": {
"runner": "nx-cloud",
"options": { "options": {
"cacheableOperations": ["build", "test", "check:types"] "cacheableOperations": ["build", "test", "check:types"]
} }

29
package.json
View File

@ -9,6 +9,7 @@
"@types/node": "20.10.0", "@types/node": "20.10.0",
"@types/proper-lockfile": "^4.1.4", "@types/proper-lockfile": "^4.1.4",
"@typescript-eslint/parser": "6.9.0", "@typescript-eslint/parser": "6.9.0",
"depcheck": "^1.4.7",
"esbuild": "^0.18.17", "esbuild": "^0.18.17",
"esbuild-node-externals": "^1.14.0", "esbuild-node-externals": "^1.14.0",
"eslint": "^8.52.0", "eslint": "^8.52.0",
@ -24,22 +25,22 @@
"prettier": "2.8.8", "prettier": "2.8.8",
"prettier-plugin-svelte": "^2.3.0", "prettier-plugin-svelte": "^2.3.0",
"proper-lockfile": "^4.1.2", "proper-lockfile": "^4.1.2",
"svelte": "^4.2.10", "svelte": "4.2.19",
"svelte-eslint-parser": "^0.33.1", "svelte-eslint-parser": "^0.33.1",
"typescript": "5.5.2", "typescript": "5.5.2",
"typescript-eslint": "^7.3.1", "typescript-eslint": "^7.3.1",
"yargs": "^17.7.2" "yargs": "^17.7.2",
"cross-spawn": "7.0.6"
}, },
"scripts": { "scripts": {
"get-past-client-version": "node scripts/getPastClientVersion.js", "get-past-client-version": "node scripts/getPastClientVersion.js",
"setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
"build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", "build": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
"build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker", "build:apps": "DISABLE_V8_COMPILE_CACHE=1 yarn build --scope @budibase/server --scope @budibase/worker",
"build:oss": "DISABLE_V8_COMPILE_CACHE=1 NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui",
"build:cli": "yarn build --scope @budibase/cli", "build:cli": "yarn build --scope @budibase/cli",
"build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal-server --scope @budibase/account-portal-ui",
"build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
"check:types": "lerna run --concurrency 2 check:types --ignore @budibase/account-portal-server", "check:types": "yarn check:dependencies && lerna run --concurrency 2 check:types",
"check:dependencies": "lerna run --concurrency 2 check:dependencies",
"build:sdk": "lerna run --stream build:sdk", "build:sdk": "lerna run --stream build:sdk",
"deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
"release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset", "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",
@ -52,15 +53,12 @@
"kill-server": "kill-port 4001 4002", "kill-server": "kill-port 4001 4002",
"kill-accountportal": "kill-port 3001 4003", "kill-accountportal": "kill-port 3001 4003",
"kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal", "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal",
"dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev",
"dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
"dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server", "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
"dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
"dev:camunda": "./scripts/deploy-camunda.sh",
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "./scripts/devDocker.sh", "dev:docker": "./scripts/devDocker.sh",
"test": "lerna run --concurrency 1 --stream test --stream", "test": "lerna run --concurrency 1 --stream test",
"test:containers:kill": "./scripts/killTestcontainers.sh", "test:containers:kill": "./scripts/killTestcontainers.sh",
"lint:eslint": "eslint packages --max-warnings=0", "lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
@ -98,9 +96,7 @@
}, },
"workspaces": { "workspaces": {
"packages": [ "packages": [
"packages/*", "packages/*"
"!packages/account-portal",
"packages/account-portal/packages/*"
] ]
}, },
"resolutions": { "resolutions": {
@ -114,7 +110,7 @@
"semver": "7.5.3", "semver": "7.5.3",
"http-cache-semantics": "4.1.1", "http-cache-semantics": "4.1.1",
"msgpackr": "1.10.1", "msgpackr": "1.10.1",
"axios": "1.6.3", "axios": "1.7.7",
"xml2js": "0.6.2", "xml2js": "0.6.2",
"unset-value": "2.0.1", "unset-value": "2.0.1",
"passport": "0.6.0", "passport": "0.6.0",
@ -124,6 +120,5 @@
}, },
"engines": { "engines": {
"node": ">=20.0.0 <21.0.0" "node": ">=20.0.0 <21.0.0"
}, }
"dependencies": {}
} }

1
packages/account-portal

@ -1 +0,0 @@
Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140

4
packages/backend-core/.npmignore
View File

@ -1,6 +1,4 @@
* *
!dist/**/* !dist/**/*
dist/tsconfig.build.tsbuildinfo dist/tsconfig.build.tsbuildinfo
!package.json !package.json
!src/**
!tests/**

28
packages/backend-core/package.json
View File

@ -9,6 +9,13 @@
"./tests": "./dist/tests/index.js", "./tests": "./dist/tests/index.js",
"./*": "./dist/*.js" "./*": "./dist/*.js"
}, },
"typesVersions": {
"*": {
"tests": [
"dist/tests/index.d.ts"
]
}
},
"author": "Budibase", "author": "Budibase",
"license": "GPL-3.0", "license": "GPL-3.0",
"scripts": { "scripts": {
@ -17,6 +24,7 @@
"build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js", "build": "tsc -p tsconfig.build.json --paths null && node ./scripts/build.js",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
"check:dependencies": "node ../../scripts/depcheck.js",
"test": "bash scripts/test.sh", "test": "bash scripts/test.sh",
"test:watch": "jest --watchAll" "test:watch": "jest --watchAll"
}, },
@ -25,17 +33,21 @@
"@budibase/pouchdb-replication-stream": "1.2.11", "@budibase/pouchdb-replication-stream": "1.2.11",
"@budibase/shared-core": "0.0.0", "@budibase/shared-core": "0.0.0",
"@budibase/types": "0.0.0", "@budibase/types": "0.0.0",
"@techpass/passport-openidconnect": "0.3.3",
"aws-cloudfront-sign": "3.0.2", "aws-cloudfront-sign": "3.0.2",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bull": "4.10.1", "bull": "4.10.1",
"correlation-id": "4.0.0", "correlation-id": "4.0.0",
"dd-trace": "5.2.0", "dd-trace": "5.26.0",
"dotenv": "16.0.1", "dotenv": "16.0.1",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"joi": "17.6.0", "joi": "17.6.0",
"jsonwebtoken": "9.0.2", "jsonwebtoken": "9.0.2",
"knex": "2.4.2",
"koa-passport": "^6.0.0", "koa-passport": "^6.0.0",
"koa-pino-logger": "4.0.0", "koa-pino-logger": "4.0.0",
"lodash": "4.17.21", "lodash": "4.17.21",
@ -46,17 +58,17 @@
"pino": "8.11.0", "pino": "8.11.0",
"pino-http": "8.3.3", "pino-http": "8.3.3",
"posthog-node": "4.0.1", "posthog-node": "4.0.1",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redlock": "4.2.0", "redlock": "4.2.0",
"rotating-file-stream": "3.1.0", "rotating-file-stream": "3.1.0",
"sanitize-s3-objectkey": "0.0.1", "sanitize-s3-objectkey": "0.0.1",
"semver": "^7.5.4", "semver": "^7.5.4",
"tar-fs": "2.1.1", "tar-fs": "2.1.1",
"uuid": "^8.3.2", "uuid": "^8.3.2"
"knex": "2.4.2"
}, },
"devDependencies": { "devDependencies": {
"@jest/types": "^29.6.3",
"@shopify/jest-koa-mocks": "5.1.1", "@shopify/jest-koa-mocks": "5.1.1",
"@swc/core": "1.3.71", "@swc/core": "1.3.71",
"@swc/jest": "0.2.27", "@swc/jest": "0.2.27",
@ -64,8 +76,9 @@
"@types/cookies": "0.7.8", "@types/cookies": "0.7.8",
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/pouchdb": "6.4.0", "@types/pouchdb": "6.4.2",
"@types/redlock": "4.0.7", "@types/redlock": "4.0.7",
"@types/semver": "7.3.7", "@types/semver": "7.3.7",
"@types/tar-fs": "2.0.1", "@types/tar-fs": "2.0.1",
@ -74,6 +87,7 @@
"ioredis-mock": "8.9.0", "ioredis-mock": "8.9.0",
"jest": "29.7.0", "jest": "29.7.0",
"jest-serial-runner": "1.2.1", "jest-serial-runner": "1.2.1",
"nock": "^13.5.6",
"pino-pretty": "10.0.0", "pino-pretty": "10.0.0",
"pouchdb-adapter-memory": "7.2.2", "pouchdb-adapter-memory": "7.2.2",
"testcontainers": "^10.7.2", "testcontainers": "^10.7.2",

4
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -190,7 +190,7 @@ export class DatabaseImpl implements Database {
} }
} }
private async performCall<T>(call: DBCallback<T>): Promise<any> { private async performCall<T>(call: DBCallback<T>): Promise<T> {
const db = this.getDb() const db = this.getDb()
const fnc = await call(db) const fnc = await call(db)
try { try {
@ -467,7 +467,7 @@ export class DatabaseImpl implements Database {
} catch (err: any) { } catch (err: any) {
// didn't exist, don't worry // didn't exist, don't worry
if (err.statusCode === 404) { if (err.statusCode === 404) {
return return { ok: true }
} else { } else {
throw new CouchDBError(err.message, err) throw new CouchDBError(err.message, err)
} }

125
packages/backend-core/src/db/instrumentation.ts
View File

@ -27,7 +27,7 @@ export class DDInstrumentedDatabase implements Database {
exists(docId?: string): Promise<boolean> { exists(docId?: string): Promise<boolean> {
return tracer.trace("db.exists", span => { return tracer.trace("db.exists", span => {
span?.addTags({ db_name: this.name, doc_id: docId }) span.addTags({ db_name: this.name, doc_id: docId })
if (docId) { if (docId) {
return this.db.exists(docId) return this.db.exists(docId)
} }
@ -37,15 +37,17 @@ export class DDInstrumentedDatabase implements Database {
get<T extends Document>(id?: string | undefined): Promise<T> { get<T extends Document>(id?: string | undefined): Promise<T> {
return tracer.trace("db.get", span => { return tracer.trace("db.get", span => {
span?.addTags({ db_name: this.name, doc_id: id }) span.addTags({ db_name: this.name, doc_id: id })
return this.db.get(id) return this.db.get(id)
}) })
} }
tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> { tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
return tracer.trace("db.tryGet", span => { return tracer.trace("db.tryGet", async span => {
span?.addTags({ db_name: this.name, doc_id: id }) span.addTags({ db_name: this.name, doc_id: id })
return this.db.tryGet(id) const doc = await this.db.tryGet<T>(id)
span.addTags({ doc_found: doc !== undefined })
return doc
}) })
} }
@ -53,13 +55,15 @@ export class DDInstrumentedDatabase implements Database {
ids: string[], ids: string[],
opts?: { allowMissing?: boolean | undefined } | undefined opts?: { allowMissing?: boolean | undefined } | undefined
): Promise<T[]> { ): Promise<T[]> {
return tracer.trace("db.getMultiple", span => { return tracer.trace("db.getMultiple", async span => {
span?.addTags({ span.addTags({
db_name: this.name, db_name: this.name,
num_docs: ids.length, num_docs: ids.length,
allow_missing: opts?.allowMissing, allow_missing: opts?.allowMissing,
}) })
return this.db.getMultiple(ids, opts) const docs = await this.db.getMultiple<T>(ids, opts)
span.addTags({ num_docs_found: docs.length })
return docs
}) })
} }
@ -69,12 +73,14 @@ export class DDInstrumentedDatabase implements Database {
idOrDoc: string | Document, idOrDoc: string | Document,
rev?: string rev?: string
): Promise<DocumentDestroyResponse> { ): Promise<DocumentDestroyResponse> {
return tracer.trace("db.remove", span => { return tracer.trace("db.remove", async span => {
span?.addTags({ db_name: this.name, doc_id: idOrDoc }) span.addTags({ db_name: this.name, doc_id: idOrDoc, rev })
const isDocument = typeof idOrDoc === "object" const isDocument = typeof idOrDoc === "object"
const id = isDocument ? idOrDoc._id! : idOrDoc const id = isDocument ? idOrDoc._id! : idOrDoc
rev = isDocument ? idOrDoc._rev : rev rev = isDocument ? idOrDoc._rev : rev
return this.db.remove(id, rev) const resp = await this.db.remove(id, rev)
span.addTags({ ok: resp.ok })
return resp
}) })
} }
@ -83,7 +89,11 @@ export class DDInstrumentedDatabase implements Database {
opts?: { silenceErrors?: boolean } opts?: { silenceErrors?: boolean }
): Promise<void> { ): Promise<void> {
return tracer.trace("db.bulkRemove", span => { return tracer.trace("db.bulkRemove", span => {
span?.addTags({ db_name: this.name, num_docs: documents.length }) span.addTags({
db_name: this.name,
num_docs: documents.length,
silence_errors: opts?.silenceErrors,
})
return this.db.bulkRemove(documents, opts) return this.db.bulkRemove(documents, opts)
}) })
} }
@ -92,15 +102,21 @@ export class DDInstrumentedDatabase implements Database {
document: AnyDocument, document: AnyDocument,
opts?: DatabasePutOpts | undefined opts?: DatabasePutOpts | undefined
): Promise<DocumentInsertResponse> { ): Promise<DocumentInsertResponse> {
return tracer.trace("db.put", span => { return tracer.trace("db.put", async span => {
span?.addTags({ db_name: this.name, doc_id: document._id }) span.addTags({
return this.db.put(document, opts) db_name: this.name,
doc_id: document._id,
force: opts?.force,
})
const resp = await this.db.put(document, opts)
span.addTags({ ok: resp.ok })
return resp
}) })
} }
bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> { bulkDocs(documents: AnyDocument[]): Promise<DocumentBulkResponse[]> {
return tracer.trace("db.bulkDocs", span => { return tracer.trace("db.bulkDocs", span => {
span?.addTags({ db_name: this.name, num_docs: documents.length }) span.addTags({ db_name: this.name, num_docs: documents.length })
return this.db.bulkDocs(documents) return this.db.bulkDocs(documents)
}) })
} }
@ -108,9 +124,15 @@ export class DDInstrumentedDatabase implements Database {
allDocs<T extends Document | RowValue>( allDocs<T extends Document | RowValue>(
params: DatabaseQueryOpts params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> { ): Promise<AllDocsResponse<T>> {
return tracer.trace("db.allDocs", span => { return tracer.trace("db.allDocs", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, ...params })
return this.db.allDocs(params) const resp = await this.db.allDocs<T>(params)
span.addTags({
total_rows: resp.total_rows,
rows_length: resp.rows.length,
offset: resp.offset,
})
return resp
}) })
} }
@ -118,57 +140,75 @@ export class DDInstrumentedDatabase implements Database {
viewName: string, viewName: string,
params: DatabaseQueryOpts params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> { ): Promise<AllDocsResponse<T>> {
return tracer.trace("db.query", span => { return tracer.trace("db.query", async span => {
span?.addTags({ db_name: this.name, view_name: viewName }) span.addTags({ db_name: this.name, view_name: viewName, ...params })
return this.db.query(viewName, params) const resp = await this.db.query<T>(viewName, params)
span.addTags({
total_rows: resp.total_rows,
rows_length: resp.rows.length,
offset: resp.offset,
})
return resp
}) })
} }
destroy(): Promise<void | OkResponse> { destroy(): Promise<OkResponse> {
return tracer.trace("db.destroy", span => { return tracer.trace("db.destroy", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.destroy() const resp = await this.db.destroy()
span.addTags({ ok: resp.ok })
return resp
}) })
} }
compact(): Promise<void | OkResponse> { compact(): Promise<OkResponse> {
return tracer.trace("db.compact", span => { return tracer.trace("db.compact", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.compact() const resp = await this.db.compact()
span.addTags({ ok: resp.ok })
return resp
}) })
} }
dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> { dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise<any> {
return tracer.trace("db.dump", span => { return tracer.trace("db.dump", span => {
span?.addTags({ db_name: this.name }) span.addTags({
db_name: this.name,
batch_limit: opts?.batch_limit,
batch_size: opts?.batch_size,
style: opts?.style,
timeout: opts?.timeout,
num_doc_ids: opts?.doc_ids?.length,
view: opts?.view,
})
return this.db.dump(stream, opts) return this.db.dump(stream, opts)
}) })
} }
load(...args: any[]): Promise<any> { load(...args: any[]): Promise<any> {
return tracer.trace("db.load", span => { return tracer.trace("db.load", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.load(...args) return this.db.load(...args)
}) })
} }
createIndex(...args: any[]): Promise<any> { createIndex(...args: any[]): Promise<any> {
return tracer.trace("db.createIndex", span => { return tracer.trace("db.createIndex", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.createIndex(...args) return this.db.createIndex(...args)
}) })
} }
deleteIndex(...args: any[]): Promise<any> { deleteIndex(...args: any[]): Promise<any> {
return tracer.trace("db.deleteIndex", span => { return tracer.trace("db.deleteIndex", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.deleteIndex(...args) return this.db.deleteIndex(...args)
}) })
} }
getIndexes(...args: any[]): Promise<any> { getIndexes(...args: any[]): Promise<any> {
return tracer.trace("db.getIndexes", span => { return tracer.trace("db.getIndexes", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_args: args.length })
return this.db.getIndexes(...args) return this.db.getIndexes(...args)
}) })
} }
@ -177,22 +217,27 @@ export class DDInstrumentedDatabase implements Database {
sql: string, sql: string,
parameters?: SqlQueryBinding parameters?: SqlQueryBinding
): Promise<T[]> { ): Promise<T[]> {
return tracer.trace("db.sql", span => { return tracer.trace("db.sql", async span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name, num_bindings: parameters?.length })
return this.db.sql(sql, parameters) const resp = await this.db.sql<T>(sql, parameters)
span.addTags({ num_rows: resp.length })
return resp
}) })
} }
sqlPurgeDocument(docIds: string[] | string): Promise<void> { sqlPurgeDocument(docIds: string[] | string): Promise<void> {
return tracer.trace("db.sqlPurgeDocument", span => { return tracer.trace("db.sqlPurgeDocument", span => {
span?.addTags({ db_name: this.name }) span.addTags({
db_name: this.name,
num_docs: Array.isArray(docIds) ? docIds.length : 1,
})
return this.db.sqlPurgeDocument(docIds) return this.db.sqlPurgeDocument(docIds)
}) })
} }
sqlDiskCleanup(): Promise<void> { sqlDiskCleanup(): Promise<void> {
return tracer.trace("db.sqlDiskCleanup", span => { return tracer.trace("db.sqlDiskCleanup", span => {
span?.addTags({ db_name: this.name }) span.addTags({ db_name: this.name })
return this.db.sqlDiskCleanup() return this.db.sqlDiskCleanup()
}) })
} }

7
packages/backend-core/src/environment.ts
View File

@ -19,6 +19,12 @@ function isDev() {
return process.env.NODE_ENV !== "production" return process.env.NODE_ENV !== "production"
} }
function parseIntSafe(number?: string) {
if (number) {
return parseInt(number)
}
}
let LOADED = false let LOADED = false
if (!LOADED && isDev() && !isTest()) { if (!LOADED && isDev() && !isTest()) {
require("dotenv").config() require("dotenv").config()
@ -231,6 +237,7 @@ const environment = {
MIN_VERSION_WITHOUT_POWER_ROLE: MIN_VERSION_WITHOUT_POWER_ROLE:
process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0", process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY, DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
BSON_BUFFER_SIZE: parseIntSafe(process.env.BSON_BUFFER_SIZE),
} }
export function setEnv(newEnvVars: Partial<typeof environment>): () => void { export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

6
packages/bbui/src/Button/Button.svelte
View File

@ -3,7 +3,7 @@
import AbsTooltip from "../Tooltip/AbsTooltip.svelte" import AbsTooltip from "../Tooltip/AbsTooltip.svelte"
import { createEventDispatcher } from "svelte" import { createEventDispatcher } from "svelte"
export let type export let type = undefined
export let disabled = false export let disabled = false
export let size = "M" export let size = "M"
export let cta = false export let cta = false
@ -16,8 +16,8 @@
export let active = false export let active = false
export let tooltip = undefined export let tooltip = undefined
export let newStyles = true export let newStyles = true
export let id export let id = undefined
export let ref export let ref = undefined
export let reverse = false export let reverse = false
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()

7
packages/bbui/src/Form/Core/DateRangePicker.svelte
View File

@ -2,13 +2,6 @@
import CoreDatePicker from "./DatePicker/DatePicker.svelte" import CoreDatePicker from "./DatePicker/DatePicker.svelte"
import Icon from "../../Icon/Icon.svelte" import Icon from "../../Icon/Icon.svelte"
export let value = null
export let disabled = false
export let readonly = false
export let error = null
export let appendTo = undefined
export let ignoreTimezones = false
let fromDate let fromDate
let toDate let toDate
</script> </script>

2
packages/bbui/src/Form/Search.svelte
View File

@ -10,7 +10,7 @@
export let disabled = false export let disabled = false
export let updateOnChange = true export let updateOnChange = true
export let quiet = false export let quiet = false
export let inputRef export let inputRef = undefined
export let helpText = null export let helpText = null
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()

8
packages/bbui/src/Form/Select.svelte
View File

@ -17,18 +17,18 @@
export let getOptionIcon = option => option?.icon export let getOptionIcon = option => option?.icon
export let getOptionColour = option => option?.colour export let getOptionColour = option => option?.colour
export let useOptionIconImage = false export let useOptionIconImage = false
export let isOptionEnabled export let isOptionEnabled = undefined
export let quiet = false export let quiet = false
export let autoWidth = false export let autoWidth = false
export let sort = false export let sort = false
export let tooltip = "" export let tooltip = ""
export let autocomplete = false export let autocomplete = false
export let customPopoverHeight export let customPopoverHeight = undefined
export let align export let align = undefined
export let footer = null export let footer = null
export let tag = null export let tag = null
export let helpText = null export let helpText = null
export let compare export let compare = undefined
export let onOptionMouseenter = () => {} export let onOptionMouseenter = () => {}
export let onOptionMouseleave = () => {} export let onOptionMouseleave = () => {}

2
packages/bbui/src/Table/Table.svelte
View File

@ -43,7 +43,7 @@
export let showHeaderBorder = true export let showHeaderBorder = true
export let placeholderText = "No rows found" export let placeholderText = "No rows found"
export let snippets = [] export let snippets = []
export let defaultSortColumn export let defaultSortColumn = undefined
export let defaultSortOrder = "Ascending" export let defaultSortOrder = "Ascending"
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()

4
packages/bbui/src/Typography/Heading.svelte
View File

@ -1,9 +1,9 @@
<script> <script lang="ts">
import "@spectrum-css/typography/dist/index-vars.css" import "@spectrum-css/typography/dist/index-vars.css"
// Sizes // Sizes
export let size = "M" export let size = "M"
export let textAlign export let textAlign = undefined
export let noPadding = false export let noPadding = false
export let weight = "default" // light, heavy, default export let weight = "default" // light, heavy, default
</script> </script>

3
packages/builder/.gitignore vendored
View File

@ -5,5 +5,4 @@ package-lock.json
release/ release/
dist/ dist/
routify routify
.routify/ .routify/
svelte.config.js

4
packages/builder/package.json
View File

@ -4,7 +4,8 @@
"license": "GPL-3.0", "license": "GPL-3.0",
"private": true, "private": true,
"scripts": { "scripts": {
"build": "routify -b && vite build --emptyOutDir", "svelte-check": "svelte-check --no-tsconfig",
"build": "yarn svelte-check && routify -b && vite build --emptyOutDir",
"start": "routify -c rollup", "start": "routify -c rollup",
"dev": "routify -c dev:vite", "dev": "routify -c dev:vite",
"dev:vite": "vite --host 0.0.0.0", "dev:vite": "vite --host 0.0.0.0",
@ -97,6 +98,7 @@
"jest": "29.7.0", "jest": "29.7.0",
"jsdom": "^21.1.1", "jsdom": "^21.1.1",
"resize-observer-polyfill": "^1.5.1", "resize-observer-polyfill": "^1.5.1",
"svelte-check": "^4.1.0",
"svelte-jester": "^1.3.2", "svelte-jester": "^1.3.2",
"vite": "^4.5.0", "vite": "^4.5.0",
"vite-plugin-static-copy": "^0.17.0", "vite-plugin-static-copy": "^0.17.0",

2
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -114,7 +114,7 @@
$: schemaFields = search.getFields( $: schemaFields = search.getFields(
$tables.list, $tables.list,
Object.values(schema || {}), Object.values(schema || {}),
{ allowLinks: true } { allowLinks: false }
) )
$: queryLimit = tableId?.includes("datasource") ? "∞" : "1000" $: queryLimit = tableId?.includes("datasource") ? "∞" : "1000"
$: isTrigger = $memoBlock?.type === AutomationStepType.TRIGGER $: isTrigger = $memoBlock?.type === AutomationStepType.TRIGGER

7
packages/builder/src/dataBinding.js
View File

@ -1141,10 +1141,11 @@ export const buildFormSchema = (component, asset) => {
const fieldSetting = settings.find( const fieldSetting = settings.find(
setting => setting.key === "field" && setting.type.startsWith("field/") setting => setting.key === "field" && setting.type.startsWith("field/")
) )
if (fieldSetting && component.field) { if (fieldSetting) {
const type = fieldSetting.type.split("field/")[1] const type = fieldSetting.type.split("field/")[1]
if (type) { const key = component.field || component._instanceName
schema[component.field] = { type } if (type && key) {
schema[key] = { type }
} }
} }
component._children?.forEach(child => { component._children?.forEach(child => {

3
packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/index.svelte
View File

@ -117,7 +117,4 @@
align-items: center; align-items: center;
margin-bottom: 12px; margin-bottom: 12px;
} }
.tabs {
}
</style> </style>

2
packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/DNDPositionIndicator.svelte
View File

@ -50,8 +50,6 @@
border-radius: 4px; border-radius: 4px;
pointer-events: none; pointer-events: none;
} }
.indicator.above {
}
.indicator.below { .indicator.below {
margin-top: 32px; margin-top: 32px;
} }

22
packages/builder/src/pages/builder/portal/plugins/index.svelte
View File

@ -1,4 +1,4 @@
<script> <script lang="ts">
import { import {
Layout, Layout,
Heading, Heading,
@ -42,23 +42,25 @@
{ column: "edit", component: EditPluginRenderer }, { column: "edit", component: EditPluginRenderer },
] ]
let modal let modal: any
let searchTerm = "" let searchTerm: any = ""
let filter = "all" let filter: any = "all"
let filterOptions = [ let filterOptions = [
{ label: "All plugins", value: "all" }, { label: "All plugins", value: "all" },
{ label: "Components", value: "component" }, { label: "Components", value: "component" },
] ]
const searchPlaceholder: any = "Search"
if (!$admin.cloud) { if (!$admin.cloud) {
filterOptions.push({ label: "Datasources", value: "datasource" }) filterOptions.push({ label: "Datasources", value: "datasource" })
} }
$: filteredPlugins = $plugins $: filteredPlugins = $plugins
.filter(plugin => { .filter((plugin: any) => {
return filter === "all" || plugin.schema.type === filter return filter === "all" || plugin.schema.type === filter
}) })
.filter(plugin => { .filter((plugin: any) => {
return ( return (
!searchTerm || !searchTerm ||
plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase()) plugin?.name?.toLowerCase().includes(searchTerm.toLowerCase())
@ -85,8 +87,8 @@
<Button <Button
on:click={() => on:click={() =>
window window
.open("https://github.com/Budibase/plugins", "_blank") ?.open("https://github.com/Budibase/plugins", "_blank")
.focus()} ?.focus()}
secondary secondary
> >
GitHub repo GitHub repo
@ -98,12 +100,12 @@
<div class="select"> <div class="select">
<Select <Select
bind:value={filter} bind:value={filter}
placeholder={null} placeholder={undefined}
options={filterOptions} options={filterOptions}
autoWidth autoWidth
/> />
</div> </div>
<Search bind:value={searchTerm} placeholder="Search" /> <Search bind:value={searchTerm} placeholder={searchPlaceholder} />
</div> </div>
{/if} {/if}
</div> </div>

25
packages/builder/src/stores/builder/automations.js
View File

@ -16,6 +16,7 @@ import {
AutomationTriggerStepId, AutomationTriggerStepId,
AutomationEventType, AutomationEventType,
AutomationStepType, AutomationStepType,
AutomationActionStepId,
} from "@budibase/types" } from "@budibase/types"
import { ActionStepID } from "constants/backend/automations" import { ActionStepID } from "constants/backend/automations"
import { FIELDS } from "constants/backend" import { FIELDS } from "constants/backend"
@ -466,9 +467,13 @@ const automationActions = store => ({
.getPathSteps(block.pathTo, automation) .getPathSteps(block.pathTo, automation)
.slice(0, -1) .slice(0, -1)
// Current step will always be the last step of the path
const currentBlock = store.actions
.getPathSteps(block.pathTo, automation)
.at(-1)
// Extract all outputs from all previous steps as available bindingsx§x // Extract all outputs from all previous steps as available bindingsx§x
let bindings = [] let bindings = []
const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => { const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
if (!name) return if (!name) return
const runtimeBinding = determineRuntimeBinding( const runtimeBinding = determineRuntimeBinding(
@ -519,9 +524,24 @@ const automationActions = store => ({
runtimeName = `loop.${name}` runtimeName = `loop.${name}`
} else if (idx === 0) { } else if (idx === 0) {
runtimeName = `trigger.${name}` runtimeName = `trigger.${name}`
} else if (
currentBlock?.stepId === AutomationActionStepId.EXECUTE_SCRIPT
) {
const stepId = pathSteps[idx].id
if (!stepId) {
notifications.error("Error generating binding: Step ID not found.")
return null
}
runtimeName = `steps["${stepId}"].${name}`
} else { } else {
runtimeName = `steps.${pathSteps[idx]?.id}.${name}` const stepId = pathSteps[idx].id
if (!stepId) {
notifications.error("Error generating binding: Step ID not found.")
return null
}
runtimeName = `steps.${stepId}.${name}`
} }
return runtimeName return runtimeName
} }
@ -637,7 +657,6 @@ const automationActions = store => ({
console.error("Loop block missing.") console.error("Loop block missing.")
} }
} }
Object.entries(schema).forEach(([name, value]) => { Object.entries(schema).forEach(([name, value]) => {
addBinding(name, value, icon, blockIdx, isLoopBlock, bindingName) addBinding(name, value, icon, blockIdx, isLoopBlock, bindingName)
}) })

7
packages/builder/svelte.config.js Normal file
View File

@ -0,0 +1,7 @@
const { vitePreprocess } = require("@sveltejs/vite-plugin-svelte")
const config = {
preprocess: vitePreprocess(),
}
module.exports = config

222
packages/client/manifest.json
View File

@ -3096,7 +3096,6 @@
"name": "Text Field", "name": "Text Field",
"icon": "Text", "icon": "Text",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3106,8 +3105,7 @@
{ {
"type": "field/string", "type": "field/string",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3226,13 +3224,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"numberfield": { "numberfield": {
"name": "Number Field", "name": "Number Field",
"icon": "123", "icon": "123",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3242,8 +3249,7 @@
{ {
"type": "field/number", "type": "field/number",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3328,13 +3334,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "number"
}
]
}
}, },
"bigintfield": { "bigintfield": {
"name": "BigInt Field", "name": "BigInt Field",
"icon": "TagBold", "icon": "TagBold",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3344,8 +3359,7 @@
{ {
"type": "field/bigint", "type": "field/bigint",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3414,13 +3428,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "number"
}
]
}
}, },
"passwordfield": { "passwordfield": {
"name": "Password Field", "name": "Password Field",
"icon": "LockClosed", "icon": "LockClosed",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3430,8 +3453,7 @@
{ {
"type": "field/string", "type": "field/string",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3500,13 +3522,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"optionsfield": { "optionsfield": {
"name": "Options Picker", "name": "Options Picker",
"icon": "Menu", "icon": "Menu",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3516,8 +3547,7 @@
{ {
"type": "field/options", "type": "field/options",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3714,13 +3744,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"multifieldselect": { "multifieldselect": {
"name": "Multi-select Picker", "name": "Multi-select Picker",
"icon": "ViewList", "icon": "ViewList",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -3730,8 +3769,7 @@
{ {
"type": "field/array", "type": "field/array",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -3922,13 +3960,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "array"
}
]
}
}, },
"booleanfield": { "booleanfield": {
"name": "Checkbox", "name": "Checkbox",
"icon": "SelectBox", "icon": "SelectBox",
"editable": true, "editable": true,
"requiredAncestors": ["form"],
"size": { "size": {
"width": 400, "width": 400,
"height": 60 "height": 60
@ -3937,8 +3984,7 @@
{ {
"type": "field/boolean", "type": "field/boolean",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -4047,13 +4093,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "boolean"
}
]
}
}, },
"longformfield": { "longformfield": {
"name": "Long Form Field", "name": "Long Form Field",
"icon": "TextAlignLeft", "icon": "TextAlignLeft",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -4063,8 +4118,7 @@
{ {
"type": "field/longform", "type": "field/longform",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -4171,13 +4225,22 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"datetimefield": { "datetimefield": {
"name": "Date Picker", "name": "Date Picker",
"icon": "Date", "icon": "Date",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -4187,8 +4250,7 @@
{ {
"type": "field/datetime", "type": "field/datetime",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -4291,7 +4353,17 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "datetime"
}
]
}
}, },
"codescanner": { "codescanner": {
"name": "Barcode/QR Scanner", "name": "Barcode/QR Scanner",
@ -4305,8 +4377,7 @@
{ {
"type": "field/barcodeqr", "type": "field/barcodeqr",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -4451,7 +4522,17 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"signaturesinglefield": { "signaturesinglefield": {
"name": "Signature", "name": "Signature",
@ -4924,7 +5005,6 @@
"icon": "Brackets", "icon": "Brackets",
"styles": ["size"], "styles": ["size"],
"editable": true, "editable": true,
"requiredAncestors": ["form"],
"size": { "size": {
"width": 400, "width": 400,
"height": 100 "height": 100
@ -4933,8 +5013,7 @@
{ {
"type": "field/json", "type": "field/json",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -5014,7 +5093,17 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
}, },
"s3upload": { "s3upload": {
"name": "S3 File Upload", "name": "S3 File Upload",
@ -5029,8 +5118,7 @@
{ {
"type": "field/s3", "type": "field/s3",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -5075,7 +5163,17 @@
"label": "Validation", "label": "Validation",
"key": "validation" "key": "validation"
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "array"
}
]
}
}, },
"dataprovider": { "dataprovider": {
"name": "Data Provider", "name": "Data Provider",
@ -7643,7 +7741,6 @@
"name": "User List Field", "name": "User List Field",
"icon": "UserGroup", "icon": "UserGroup",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -7653,8 +7750,7 @@
{ {
"type": "field/bb_reference", "type": "field/bb_reference",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -7744,14 +7840,23 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "array"
}
]
}
}, },
"bbreferencesinglefield": { "bbreferencesinglefield": {
"devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels", "devComment": "As bb reference is only used for user subtype for now, we are using user for icon and labels",
"name": "User Field", "name": "User Field",
"icon": "User", "icon": "User",
"styles": ["size"], "styles": ["size"],
"requiredAncestors": ["form"],
"editable": true, "editable": true,
"size": { "size": {
"width": 400, "width": 400,
@ -7761,8 +7866,7 @@
{ {
"type": "field/bb_reference_single", "type": "field/bb_reference_single",
"label": "Field", "label": "Field",
"key": "field", "key": "field"
"required": true
}, },
{ {
"type": "text", "type": "text",
@ -7852,6 +7956,16 @@
} }
] ]
} }
] ],
"context": {
"type": "static",
"values": [
{
"label": "Value",
"key": "value",
"type": "string"
}
]
}
} }
} }

2
packages/client/src/components/app/charts/AreaChart.svelte
View File

@ -18,8 +18,6 @@
export let palette export let palette
export let c1, c2, c3, c4, c5 export let c1, c2, c3, c4, c5
// Area specific props
export let area
export let stacked export let stacked
export let gradient export let gradient

145
packages/client/src/components/app/forms/Field.svelte
View File

@ -1,7 +1,10 @@
<script> <script>
import Placeholder from "../Placeholder.svelte"
import { getContext, onDestroy } from "svelte" import { getContext, onDestroy } from "svelte"
import { writable } from "svelte/store"
import { Icon } from "@budibase/bbui" import { Icon } from "@budibase/bbui"
import { memo } from "@budibase/frontend-core"
import Placeholder from "../Placeholder.svelte"
import InnerForm from "./InnerForm.svelte"
export let label export let label
export let field export let field
@ -20,26 +23,39 @@
const formContext = getContext("form") const formContext = getContext("form")
const formStepContext = getContext("form-step") const formStepContext = getContext("form-step")
const fieldGroupContext = getContext("field-group") const fieldGroupContext = getContext("field-group")
const { styleable, builderStore } = getContext("sdk") const { styleable, builderStore, Provider } = getContext("sdk")
const component = getContext("component") const component = getContext("component")
// Register field with form // Register field with form
const formApi = formContext?.formApi const formApi = formContext?.formApi
const labelPos = fieldGroupContext?.labelPosition || "above" const labelPos = fieldGroupContext?.labelPosition || "above"
let formField
let touched = false let touched = false
let labelNode let labelNode
$: formStep = formStepContext ? $formStepContext || 1 : 1 // Memoize values required to register the field to avoid loops
$: formField = formApi?.registerField( const formStep = formStepContext || writable(1)
field, const fieldInfo = memo({
field: field || $component.name,
type, type,
defaultValue, defaultValue,
disabled, disabled,
readonly, readonly,
validation, validation,
formStep formStep: $formStep || 1,
) })
$: fieldInfo.set({
field: field || $component.name,
type,
defaultValue,
disabled,
readonly,
validation,
formStep: $formStep || 1,
})
$: registerField($fieldInfo)
$: schemaType = $: schemaType =
fieldSchema?.type !== "formula" && fieldSchema?.type !== "bigint" fieldSchema?.type !== "formula" && fieldSchema?.type !== "bigint"
? fieldSchema?.type ? fieldSchema?.type
@ -58,6 +74,18 @@
// Determine label class from position // Determine label class from position
$: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}` $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
const registerField = info => {
formField = formApi?.registerField(
info.field,
info.type,
info.defaultValue,
info.disabled,
info.readonly,
info.validation,
info.formStep
)
}
const updateLabel = e => { const updateLabel = e => {
if (touched) { if (touched) {
builderStore.actions.updateProp("label", e.target.textContent) builderStore.actions.updateProp("label", e.target.textContent)
@ -71,52 +99,65 @@
}) })
</script> </script>
<div <Provider data={{ value: fieldState?.value }}>
class="spectrum-Form-item" {#if !formContext}
class:span-2={span === 2} <InnerForm
class:span-3={span === 3} {disabled}
class:span-6={span === 6 || !span} {readonly}
use:styleable={$component.styles} currentStep={writable(1)}
class:above={labelPos === "above"} provideContext={false}
>
{#key $component.editing}
<label
bind:this={labelNode}
contenteditable={$component.editing}
on:blur={$component.editing ? updateLabel : null}
on:input={() => (touched = true)}
class:hidden={!label}
class:readonly
for={fieldState?.fieldId}
class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${labelClass}`}
> >
{label || " "} <svelte:self {...$$props} bind:fieldState bind:fieldApi bind:fieldSchema>
</label> <slot />
{/key} </svelte:self>
<div class="spectrum-Form-itemField"> </InnerForm>
{#if !formContext} {:else}
<Placeholder text="Form components need to be wrapped in a form" /> <div
{:else if !fieldState} class="spectrum-Form-item"
<Placeholder /> class:span-2={span === 2}
{:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)} class:span-3={span === 3}
<Placeholder class:span-6={span === 6 || !span}
text="This Field setting is the wrong data type for this component" use:styleable={$component.styles}
/> class:above={labelPos === "above"}
{:else} >
<slot /> {#key $component.editing}
{#if fieldState.error} <label
<div class="error"> bind:this={labelNode}
<Icon name="Alert" /> contenteditable={$component.editing}
<span>{fieldState.error}</span> on:blur={$component.editing ? updateLabel : null}
</div> on:input={() => (touched = true)}
{:else if helpText} class:hidden={!label}
<div class="helpText"> class:readonly
<Icon name="HelpOutline" /> <span>{helpText}</span> for={fieldState?.fieldId}
</div> class={`spectrum-FieldLabel spectrum-FieldLabel--sizeM spectrum-Form-itemLabel ${labelClass}`}
{/if} >
{/if} {label || " "}
</div> </label>
</div> {/key}
<div class="spectrum-Form-itemField">
{#if !fieldState}
<Placeholder />
{:else if schemaType && schemaType !== type && !["options", "longform"].includes(type)}
<Placeholder
text="This Field setting is the wrong data type for this component"
/>
{:else}
<slot />
{#if fieldState.error}
<div class="error">
<Icon name="Alert" />
<span>{fieldState.error}</span>
</div>
{:else if helpText}
<div class="helpText">
<Icon name="HelpOutline" /> <span>{helpText}</span>
</div>
{/if}
{/if}
</div>
</div>
{/if}
</Provider>
<style> <style>
:global(.form-block .spectrum-Form-item.span-2) { :global(.form-block .spectrum-Form-item.span-2) {

3
packages/client/src/components/app/forms/Form.svelte
View File

@ -5,7 +5,6 @@
import { writable } from "svelte/store" import { writable } from "svelte/store"
export let dataSource export let dataSource
export let theme
export let size export let size
export let disabled = false export let disabled = false
export let readonly = false export let readonly = false
@ -113,11 +112,9 @@
{#key resetKey} {#key resetKey}
<InnerForm <InnerForm
{dataSource} {dataSource}
{theme}
{size} {size}
{disabled} {disabled}
{readonly} {readonly}
{actionType}
{schema} {schema}
{definition} {definition}
{initialValues} {initialValues}

14
packages/client/src/components/app/forms/InnerForm.svelte
View File

@ -14,6 +14,10 @@
export let disableSchemaValidation = false export let disableSchemaValidation = false
export let editAutoColumns = false export let editAutoColumns = false
// For internal use only, to disable context when being used with standalone
// fields
export let provideContext = true
// We export this store so that when we remount the inner form we can still // We export this store so that when we remount the inner form we can still
// persist what step we're on // persist what step we're on
export let currentStep export let currentStep
@ -442,8 +446,14 @@
] ]
</script> </script>
<Provider {actions} data={dataContext}> {#if provideContext}
<Provider {actions} data={dataContext}>
<div use:styleable={$component.styles} class={size}>
<slot />
</div>
</Provider>
{:else}
<div use:styleable={$component.styles} class={size}> <div use:styleable={$component.styles} class={size}>
<slot /> <slot />
</div> </div>
</Provider> {/if}

27
packages/client/src/utils/blocks.js
View File

@ -1,5 +1,7 @@
import { makePropSafe as safe } from "@budibase/string-templates" import { makePropSafe as safe } from "@budibase/string-templates"
import { API } from "../api/index.js" import { API } from "../api/index.js"
import { UILogicalOperator } from "@budibase/types"
import { OnEmptyFilter } from "@budibase/frontend-core/src/constants.js"
// Map of data types to component types for search fields inside blocks // Map of data types to component types for search fields inside blocks
const schemaComponentMap = { const schemaComponentMap = {
@ -60,7 +62,11 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
* @param formId the ID of the form containing the search fields * @param formId the ID of the form containing the search fields
*/ */
export const enrichFilter = (filter, columns, formId) => { export const enrichFilter = (filter, columns, formId) => {
let enrichedFilter = [...(filter || [])] if (!columns?.length) {
return filter
}
let newFilters = []
columns?.forEach(column => { columns?.forEach(column => {
const safePath = column.name.split(".").map(safe).join(".") const safePath = column.name.split(".").map(safe).join(".")
const stringType = column.type === "string" || column.type === "formula" const stringType = column.type === "string" || column.type === "formula"
@ -69,7 +75,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For dates, use a range of the entire day selected // For dates, use a range of the entire day selected
if (dateType) { if (dateType) {
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: "rangeLow", operator: "rangeLow",
@ -79,7 +85,7 @@ export const enrichFilter = (filter, columns, formId) => {
const format = "YYYY-MM-DDTHH:mm:ss.SSSZ" const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}` let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
hbs = `{{#if ${binding} }}${hbs}{{/if}}` hbs = `{{#if ${binding} }}${hbs}{{/if}}`
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: "rangeHigh", operator: "rangeHigh",
@ -90,7 +96,7 @@ export const enrichFilter = (filter, columns, formId) => {
// For other fields, do an exact match // For other fields, do an exact match
else { else {
enrichedFilter.push({ newFilters.push({
field: column.name, field: column.name,
type: column.type, type: column.type,
operator: stringType ? "string" : "equal", operator: stringType ? "string" : "equal",
@ -99,5 +105,16 @@ export const enrichFilter = (filter, columns, formId) => {
}) })
} }
}) })
return enrichedFilter
return {
logicalOperator: UILogicalOperator.ALL,
onEmptyFilter: OnEmptyFilter.RETURN_ALL,
groups: [
...(filter?.groups || []),
{
logicalOperator: UILogicalOperator.ALL,
filters: newFilters,
},
],
}
} }

2
packages/pro

@ -1 +1 @@
Subproject commit bfeece324a03a3a5f25137bf3f8c66d5ed6103d8 Subproject commit 25dd40ee12b048307b558ebcedb36548d6e042cd

41
packages/server/package.json
View File

@ -13,6 +13,7 @@
"build": "node ./scripts/build.js", "build": "node ./scripts/build.js",
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/", "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
"check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020", "check:types": "tsc -p tsconfig.json --noEmit --paths null --target es2020",
"check:dependencies": "node ../../scripts/depcheck.js",
"build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets", "build:isolated-vm-lib:snippets": "esbuild --minify --bundle src/jsRunner/bundles/snippets.ts --outfile=src/jsRunner/bundles/snippets.ivm.bundle.js --platform=node --format=iife --global-name=snippets",
"build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers", "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
"build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson", "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
@ -49,9 +50,11 @@
"license": "GPL-3.0", "license": "GPL-3.0",
"dependencies": { "dependencies": {
"@apidevtools/swagger-parser": "10.0.3", "@apidevtools/swagger-parser": "10.0.3",
"@azure/msal-node": "^2.5.1",
"@budibase/backend-core": "0.0.0", "@budibase/backend-core": "0.0.0",
"@budibase/client": "0.0.0", "@budibase/client": "0.0.0",
"@budibase/frontend-core": "0.0.0", "@budibase/frontend-core": "0.0.0",
"@budibase/nano": "10.1.5",
"@budibase/pro": "0.0.0", "@budibase/pro": "0.0.0",
"@budibase/shared-core": "0.0.0", "@budibase/shared-core": "0.0.0",
"@budibase/string-templates": "0.0.0", "@budibase/string-templates": "0.0.0",
@ -60,15 +63,17 @@
"@bull-board/koa": "5.10.2", "@bull-board/koa": "5.10.2",
"@elastic/elasticsearch": "7.10.0", "@elastic/elasticsearch": "7.10.0",
"@google-cloud/firestore": "7.8.0", "@google-cloud/firestore": "7.8.0",
"@koa/router": "8.0.8", "@koa/cors": "5.0.0",
"@koa/router": "13.1.0",
"@socket.io/redis-adapter": "^8.2.1", "@socket.io/redis-adapter": "^8.2.1",
"@types/xml2js": "^0.4.14", "@types/xml2js": "^0.4.14",
"airtable": "0.12.2", "airtable": "0.12.2",
"arangojs": "7.2.0", "arangojs": "7.2.0",
"archiver": "7.0.1", "archiver": "7.0.1",
"aws-sdk": "2.1030.0", "aws-sdk": "2.1692.0",
"bcrypt": "5.1.0", "bcrypt": "5.1.0",
"bcryptjs": "2.4.3", "bcryptjs": "2.4.3",
"bson": "^6.9.0",
"buffer": "6.0.3", "buffer": "6.0.3",
"bull": "4.10.1", "bull": "4.10.1",
"chokidar": "3.5.3", "chokidar": "3.5.3",
@ -76,17 +81,20 @@
"cookies": "0.8.0", "cookies": "0.8.0",
"csvtojson": "2.0.10", "csvtojson": "2.0.10",
"curlconverter": "3.21.0", "curlconverter": "3.21.0",
"dd-trace": "5.2.0", "dayjs": "^1.10.8",
"dd-trace": "5.26.0",
"dotenv": "8.2.0", "dotenv": "8.2.0",
"form-data": "4.0.0", "form-data": "4.0.0",
"global-agent": "3.0.0", "global-agent": "3.0.0",
"google-auth-library": "^8.0.1",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2", "ioredis": "5.3.2",
"isolated-vm": "^4.7.2", "isolated-vm": "^4.7.2",
"jimp": "0.22.12", "jimp": "1.1.4",
"joi": "17.6.0", "joi": "17.6.0",
"js-yaml": "4.1.0", "js-yaml": "4.1.0",
"jsonschema": "1.4.0", "jsonschema": "1.4.0",
"jsonwebtoken": "9.0.2",
"knex": "2.4.2", "knex": "2.4.2",
"koa": "2.13.4", "koa": "2.13.4",
"koa-body": "4.2.0", "koa-body": "4.2.0",
@ -97,7 +105,7 @@
"lodash": "4.17.21", "lodash": "4.17.21",
"memorystream": "0.3.1", "memorystream": "0.3.1",
"mongodb": "6.7.0", "mongodb": "6.7.0",
"mssql": "10.0.1", "mssql": "11.0.1",
"mysql2": "3.9.8", "mysql2": "3.9.8",
"node-fetch": "2.6.7", "node-fetch": "2.6.7",
"object-sizeof": "2.6.1", "object-sizeof": "2.6.1",
@ -105,24 +113,28 @@
"openapi-types": "9.3.1", "openapi-types": "9.3.1",
"oracledb": "6.5.1", "oracledb": "6.5.1",
"pg": "8.10.0", "pg": "8.10.0",
"pouchdb": "7.3.0", "pouchdb": "9.0.0",
"pouchdb-all-dbs": "1.1.1", "pouchdb-all-dbs": "1.1.1",
"pouchdb-find": "7.2.2", "pouchdb-find": "9.0.0",
"redis": "4", "redis": "4",
"semver": "^7.5.4",
"serialize-error": "^7.0.1", "serialize-error": "^7.0.1",
"server-destroy": "1.0.1", "server-destroy": "1.0.1",
"snowflake-promise": "^4.5.0", "snowflake-sdk": "^1.15.0",
"socket.io": "4.7.5", "socket.io": "4.8.1",
"svelte": "^4.2.10",
"tar": "6.2.1", "tar": "6.2.1",
"tmp": "0.2.3", "tmp": "0.2.3",
"to-json-schema": "0.2.5", "to-json-schema": "0.2.5",
"uuid": "^8.3.2", "uuid": "^8.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",
"worker-farm": "1.7.0", "worker-farm": "1.7.0",
"xml2js": "0.5.0" "xml2js": "0.6.2"
}, },
"devDependencies": { "devDependencies": {
"@babel/core": "^7.22.5",
"@babel/preset-env": "7.16.11", "@babel/preset-env": "7.16.11",
"@jest/types": "^29.6.3",
"@swc/core": "1.3.71", "@swc/core": "1.3.71",
"@swc/jest": "0.2.27", "@swc/jest": "0.2.27",
"@types/archiver": "6.0.2", "@types/archiver": "6.0.2",
@ -130,19 +142,24 @@
"@types/jest": "29.5.5", "@types/jest": "29.5.5",
"@types/koa": "2.13.4", "@types/koa": "2.13.4",
"@types/koa-send": "^4.1.6", "@types/koa-send": "^4.1.6",
"@types/koa__router": "8.0.8", "@types/koa__cors": "5.0.0",
"@types/koa__router": "12.0.4",
"@types/lodash": "4.14.200", "@types/lodash": "4.14.200",
"@types/mssql": "9.1.4", "@types/mssql": "9.1.5",
"@types/node": "^22.9.0",
"@types/node-fetch": "2.6.4", "@types/node-fetch": "2.6.4",
"@types/oracledb": "6.5.1", "@types/oracledb": "6.5.1",
"@types/pg": "8.6.6", "@types/pg": "8.6.6",
"@types/pouchdb": "6.4.2",
"@types/server-destroy": "1.0.1", "@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.14", "@types/supertest": "2.0.14",
"@types/tar": "6.1.5", "@types/tar": "6.1.5",
"@types/tmp": "0.2.6", "@types/tmp": "0.2.6",
"@types/uuid": "8.3.4", "@types/uuid": "8.3.4",
"chance": "^1.1.12",
"copyfiles": "2.4.1", "copyfiles": "2.4.1",
"docker-compose": "0.23.17", "docker-compose": "0.23.17",
"ioredis-mock": "8.9.0",
"jest": "29.7.0", "jest": "29.7.0",
"jest-extended": "^4.0.2", "jest-extended": "^4.0.2",
"jest-openapi": "0.14.2", "jest-openapi": "0.14.2",

1239
packages/server/specs/openapi.json
View File

File diff suppressed because it is too large Load Diff

865
packages/server/specs/openapi.yaml
View File

@ -23,6 +23,13 @@ components:
description: The ID of the table which this request is targeting. description: The ID of the table which this request is targeting.
schema: schema:
type: string type: string
viewId:
in: path
name: viewId
required: true
description: The ID of the view which this request is targeting.
schema:
type: string
rowId: rowId:
in: path in: path
name: rowId name: rowId
@ -36,7 +43,7 @@ components:
required: true required: true
description: The ID of the app which this request is targeting. description: The ID of the app which this request is targeting.
schema: schema:
default: "{{ appId }}" default: "{{appId}}"
type: string type: string
appIdUrl: appIdUrl:
in: path in: path
@ -44,7 +51,7 @@ components:
required: true required: true
description: The ID of the app which this request is targeting. description: The ID of the app which this request is targeting.
schema: schema:
default: "{{ appId }}" default: "{{appId}}"
type: string type: string
queryId: queryId:
in: path in: path
@ -442,6 +449,74 @@ components:
# TYPE budibase_quota_limit_automations gauge # TYPE budibase_quota_limit_automations gauge
budibase_quota_limit_automations 9007199254740991 budibase_quota_limit_automations 9007199254740991
view:
value:
data:
name: peopleView
tableId: ta_896a325f7e8147d2a2cda93c5d236511
schema:
name:
visible: true
readonly: false
order: 1
width: 300
age:
visible: true
readonly: true
order: 2
width: 200
salary:
visible: false
readonly: false
query:
logicalOperator: all
onEmptyFilter: none
groups:
- logicalOperator: any
filters:
- operator: string
field: name
value: John
- operator: range
field: age
value:
low: 18
high: 100
primaryDisplay: name
views:
value:
data:
- name: peopleView
tableId: ta_896a325f7e8147d2a2cda93c5d236511
schema:
name:
visible: true
readonly: false
order: 1
width: 300
age:
visible: true
readonly: true
order: 2
width: 200
salary:
visible: false
readonly: false
query:
logicalOperator: all
onEmptyFilter: none
groups:
- logicalOperator: any
filters:
- operator: string
field: name
value: John
- operator: range
field: age
value:
low: 18
high: 100
primaryDisplay: name
securitySchemes: securitySchemes:
ApiKeyAuth: ApiKeyAuth:
type: apiKey type: apiKey
@ -761,7 +836,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -931,7 +1005,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -1108,7 +1181,6 @@ components:
enum: enum:
- static - static
- dynamic - dynamic
- ai
description: Defines whether this is a static or dynamic formula. description: Defines whether this is a static or dynamic formula.
- type: object - type: object
properties: properties:
@ -1704,6 +1776,644 @@ components:
- userIds - userIds
required: required:
- data - data
view:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able to
access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a calculation is
configured all non-calculation columns will be used for
grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
viewOutput:
type: object
properties:
data:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
- id
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able
to access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a calculation
is configured all non-calculation columns will be used
for grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
id:
description: The ID of the view.
type: string
required:
- data
viewSearch:
type: object
properties:
data:
type: array
items:
description: The view to be created/updated.
type: object
required:
- name
- schema
- tableId
- id
properties:
name:
description: The name of the view.
type: string
tableId:
description: The ID of the table this view is based on.
type: string
type:
description: The type of view - standard (empty value) or calculation.
type: string
enum:
- calculation
primaryDisplay:
type: string
description: A column used to display rows from this view - usually used when
rendered in tables.
query:
description: Search parameters for view
type: object
required: []
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
onEmptyFilter:
description: If no filters match, should the view return all rows, or no rows.
type: string
enum:
- all
- none
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
groups:
description: A grouping of filters to be applied.
type: array
items:
type: object
properties:
logicalOperator:
description: When using groups this defines whether all of the filters must
match, or only one of them.
type: string
enum:
- all
- any
filters:
description: A list of filters to apply
type: array
items:
type: object
properties:
operator:
type: string
description: The type of search operation which is being performed.
enum:
- equal
- notEqual
- empty
- notEmpty
- fuzzy
- string
- contains
- notContains
- containsAny
- oneOf
- range
field:
type: string
description: The field in the view to perform the search on.
value:
description: The value to search for - the type will depend on the operator in
use.
oneOf:
- type: string
- type: number
- type: boolean
- type: object
- type: array
sort:
type: object
required:
- field
properties:
field:
type: string
description: The field from the table/view schema to sort on.
order:
type: string
description: The order in which to sort.
enum:
- ascending
- descending
type:
type: string
description: The type of sort to perform (by number, or by alphabetically).
enum:
- string
- number
schema:
type: object
additionalProperties:
oneOf:
- type: object
properties:
visible:
type: boolean
description: Defines whether the column is visible or not - rows
retrieved/updated through this view will not be able
to access it.
readonly:
type: boolean
description: "When used in combination with 'visible: true' the column will be
visible in row responses but cannot be updated."
order:
type: integer
description: A number defining where the column shows up in tables, lowest being
first.
width:
type: integer
description: A width for the column, defined in pixels - this affects rendering
in tables.
column:
type: array
description: If this is a relationship column, we can set the columns we wish to
include
items:
type: object
properties:
readonly:
type: boolean
- type: object
properties:
calculationType:
type: string
description: This column should be built from a calculation, specifying a type
and field. It is important to note when a
calculation is configured all non-calculation
columns will be used for grouping.
enum:
- sum
- avg
- count
- min
- max
field:
type: string
description: The field from the table to perform the calculation on.
distinct:
type: boolean
description: Can be used in tandem with the count calculation type, to count
unique entries.
id:
description: The ID of the view.
type: string
required:
- data
security: security:
- ApiKeyAuth: [] - ApiKeyAuth: []
paths: paths:
@ -2136,6 +2846,32 @@ paths:
examples: examples:
search: search:
$ref: "#/components/examples/rows" $ref: "#/components/examples/rows"
"/views/{viewId}/rows/search":
post:
operationId: rowViewSearch
summary: Search for rows in a view
tags:
- rows
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/rowSearch"
responses:
"200":
description: The response will contain an array of rows that match the search
parameters.
content:
application/json:
schema:
$ref: "#/components/schemas/searchOutput"
examples:
search:
$ref: "#/components/examples/rows"
/tables: /tables:
post: post:
operationId: tableCreate operationId: tableCreate
@ -2359,4 +3095,123 @@ paths:
examples: examples:
users: users:
$ref: "#/components/examples/users" $ref: "#/components/examples/users"
/views:
post:
operationId: viewCreate
summary: Create a view
description: Create a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/appId"
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/view"
examples:
view:
$ref: "#/components/examples/view"
responses:
"200":
description: Returns the created view, including the ID which has been generated
for it.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
"/views/{viewId}":
put:
operationId: viewUpdate
summary: Update a view
description: Update a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
requestBody:
content:
application/json:
schema:
$ref: "#/components/schemas/view"
examples:
view:
$ref: "#/components/examples/view"
responses:
"200":
description: Returns the updated view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
delete:
operationId: viewDestroy
summary: Delete a view
description: Delete a view, this can be against an internal or external table.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
responses:
"200":
description: Returns the deleted view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
get:
operationId: viewGetById
summary: Retrieve a view
description: Lookup a view, this could be internal or external.
tags:
- views
parameters:
- $ref: "#/components/parameters/viewId"
- $ref: "#/components/parameters/appId"
responses:
"200":
description: Returns the retrieved view.
content:
application/json:
schema:
$ref: "#/components/schemas/viewOutput"
examples:
view:
$ref: "#/components/examples/view"
/views/search:
post:
operationId: viewSearch
summary: Search for views
description: Based on view properties (currently only name) search for views.
tags:
- views
parameters:
- $ref: "#/components/parameters/appId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/nameSearch"
responses:
"200":
description: Returns the found views, based on the search parameters.
content:
application/json:
schema:
$ref: "#/components/schemas/viewSearch"
examples:
views:
$ref: "#/components/examples/views"
tags: [] tags: []

10
packages/server/specs/parameters.ts
View File

@ -8,6 +8,16 @@ export const tableId = {
}, },
} }
export const viewId = {
in: "path",
name: "viewId",
required: true,
description: "The ID of the view which this request is targeting.",
schema: {
type: "string",
},
}
export const rowId = { export const rowId = {
in: "path", in: "path",
name: "rowId", name: "rowId",

3
packages/server/specs/resources/index.ts
View File

@ -6,6 +6,7 @@ import user from "./user"
import metrics from "./metrics" import metrics from "./metrics"
import misc from "./misc" import misc from "./misc"
import roles from "./roles" import roles from "./roles"
import view from "./view"
export const examples = { export const examples = {
...application.getExamples(), ...application.getExamples(),
@ -16,6 +17,7 @@ export const examples = {
...misc.getExamples(), ...misc.getExamples(),
...metrics.getExamples(), ...metrics.getExamples(),
...roles.getExamples(), ...roles.getExamples(),
...view.getExamples(),
} }
export const schemas = { export const schemas = {
@ -26,4 +28,5 @@ export const schemas = {
...user.getSchemas(), ...user.getSchemas(),
...misc.getSchemas(), ...misc.getSchemas(),
...roles.getSchemas(), ...roles.getSchemas(),
...view.getSchemas(),
} }

182
packages/server/specs/resources/misc.ts
View File

@ -1,99 +1,101 @@
import { object } from "./utils" import { object } from "./utils"
import Resource from "./utils/Resource" import Resource from "./utils/Resource"
export const searchSchema = {
type: "object",
properties: {
allOr: {
type: "boolean",
description:
"Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
},
string: {
type: "object",
example: {
columnName1: "value",
columnName2: "value",
},
description:
"A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
additionalProperties: {
type: "string",
description: "The value to search for in the column.",
},
},
fuzzy: {
type: "object",
description:
"Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
},
range: {
type: "object",
description:
'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
example: {
columnName1: {
low: 10,
high: 20,
},
},
},
equal: {
type: "object",
description:
"Searches for rows that have a column value that is exactly the value set.",
},
notEqual: {
type: "object",
description:
"Searches for any row which does not contain the specified column value.",
},
empty: {
type: "object",
description:
"Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
example: {
columnName1: "",
},
},
notEmpty: {
type: "object",
description: "Searches for rows which have the specified column.",
},
oneOf: {
type: "object",
description:
"Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
},
contains: {
type: "object",
description:
"Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
example: {
arrayColumn: ["a", "b"],
},
},
notContains: {
type: "object",
description:
"The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
example: {
arrayColumn: ["a", "b"],
},
},
containsAny: {
type: "object",
description:
"As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
example: {
arrayColumn: ["a", "b"],
},
},
},
}
export default new Resource().setSchemas({ export default new Resource().setSchemas({
rowSearch: object( rowSearch: object(
{ {
query: { query: searchSchema,
type: "object",
properties: {
allOr: {
type: "boolean",
description:
"Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
},
string: {
type: "object",
example: {
columnName1: "value",
columnName2: "value",
},
description:
"A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
additionalProperties: {
type: "string",
description: "The value to search for in the column.",
},
},
fuzzy: {
type: "object",
description:
"Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
},
range: {
type: "object",
description:
'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
example: {
columnName1: {
low: 10,
high: 20,
},
},
},
equal: {
type: "object",
description:
"Searches for rows that have a column value that is exactly the value set.",
},
notEqual: {
type: "object",
description:
"Searches for any row which does not contain the specified column value.",
},
empty: {
type: "object",
description:
"Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
example: {
columnName1: "",
},
},
notEmpty: {
type: "object",
description: "Searches for rows which have the specified column.",
},
oneOf: {
type: "object",
description:
"Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
},
contains: {
type: "object",
description:
"Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
example: {
arrayColumn: ["a", "b"],
},
},
notContains: {
type: "object",
description:
"The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
example: {
arrayColumn: ["a", "b"],
},
},
containsAny: {
type: "object",
description:
"As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
example: {
arrayColumn: ["a", "b"],
},
},
},
},
paginate: { paginate: {
type: "boolean", type: "boolean",
description: "Enables pagination, by default this is disabled.", description: "Enables pagination, by default this is disabled.",

274
packages/server/specs/resources/view.ts Normal file
View File

@ -0,0 +1,274 @@
import { object } from "./utils"
import Resource from "./utils/Resource"
import {
ArrayOperator,
BasicOperator,
CalculationType,
RangeOperator,
SortOrder,
SortType,
} from "@budibase/types"
import { cloneDeep } from "lodash"
const view = {
name: "peopleView",
tableId: "ta_896a325f7e8147d2a2cda93c5d236511",
schema: {
name: {
visible: true,
readonly: false,
order: 1,
width: 300,
},
age: {
visible: true,
readonly: true,
order: 2,
width: 200,
},
salary: {
visible: false,
readonly: false,
},
},
query: {
logicalOperator: "all",
onEmptyFilter: "none",
groups: [
{
logicalOperator: "any",
filters: [
{ operator: "string", field: "name", value: "John" },
{ operator: "range", field: "age", value: { low: 18, high: 100 } },
],
},
],
},
primaryDisplay: "name",
}
const baseColumnDef = {
visible: {
type: "boolean",
description:
"Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it.",
},
readonly: {
type: "boolean",
description:
"When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated.",
},
order: {
type: "integer",
description:
"A number defining where the column shows up in tables, lowest being first.",
},
width: {
type: "integer",
description:
"A width for the column, defined in pixels - this affects rendering in tables.",
},
column: {
type: "array",
description:
"If this is a relationship column, we can set the columns we wish to include",
items: {
type: "object",
properties: {
readonly: {
type: "boolean",
},
},
},
},
}
const logicalOperator = {
description:
"When using groups this defines whether all of the filters must match, or only one of them.",
type: "string",
enum: ["all", "any"],
}
const filterGroup = {
description: "A grouping of filters to be applied.",
type: "array",
items: {
type: "object",
properties: {
logicalOperator,
filters: {
description: "A list of filters to apply",
type: "array",
items: {
type: "object",
properties: {
operator: {
type: "string",
description:
"The type of search operation which is being performed.",
enum: [
...Object.values(BasicOperator),
...Object.values(ArrayOperator),
...Object.values(RangeOperator),
],
},
field: {
type: "string",
description: "The field in the view to perform the search on.",
},
value: {
description:
"The value to search for - the type will depend on the operator in use.",
oneOf: [
{ type: "string" },
{ type: "number" },
{ type: "boolean" },
{ type: "object" },
{ type: "array" },
],
},
},
},
},
},
},
}
// have to clone to avoid constantly recursive structure - we can't represent this easily
const layeredFilterGroup: any = cloneDeep(filterGroup)
layeredFilterGroup.items.properties.groups = filterGroup
const viewQuerySchema = {
description: "Search parameters for view",
type: "object",
required: [],
properties: {
logicalOperator,
onEmptyFilter: {
description:
"If no filters match, should the view return all rows, or no rows.",
type: "string",
enum: ["all", "none"],
},
groups: layeredFilterGroup,
},
}
const viewSchema = {
description: "The view to be created/updated.",
type: "object",
required: ["name", "schema", "tableId"],
properties: {
name: {
description: "The name of the view.",
type: "string",
},
tableId: {
description: "The ID of the table this view is based on.",
type: "string",
},
type: {
description: "The type of view - standard (empty value) or calculation.",
type: "string",
enum: ["calculation"],
},
primaryDisplay: {
type: "string",
description:
"A column used to display rows from this view - usually used when rendered in tables.",
},
query: viewQuerySchema,
sort: {
type: "object",
required: ["field"],
properties: {
field: {
type: "string",
description: "The field from the table/view schema to sort on.",
},
order: {
type: "string",
description: "The order in which to sort.",
enum: Object.values(SortOrder),
},
type: {
type: "string",
description:
"The type of sort to perform (by number, or by alphabetically).",
enum: Object.values(SortType),
},
},
},
schema: {
type: "object",
additionalProperties: {
oneOf: [
{
type: "object",
properties: baseColumnDef,
},
{
type: "object",
properties: {
calculationType: {
type: "string",
description:
"This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.",
enum: Object.values(CalculationType),
},
field: {
type: "string",
description:
"The field from the table to perform the calculation on.",
},
distinct: {
type: "boolean",
description:
"Can be used in tandem with the count calculation type, to count unique entries.",
},
},
},
],
},
},
},
}
const viewOutputSchema = {
...viewSchema,
properties: {
...viewSchema.properties,
id: {
description: "The ID of the view.",
type: "string",
},
},
required: [...viewSchema.required, "id"],
}
export default new Resource()
.setExamples({
view: {
value: {
data: view,
},
},
views: {
value: {
data: [view],
},
},
})
.setSchemas({
view: viewSchema,
viewOutput: object({
data: viewOutputSchema,
}),
viewSearch: object({
data: {
type: "array",
items: viewOutputSchema,
},
}),
})

3
packages/server/src/api/controllers/public/mapping/applications.ts
View File

@ -1,6 +1,7 @@
import { Application } from "./types" import { Application } from "./types"
import { RequiredKeys } from "@budibase/types"
function application(body: any): Application { function application(body: any): RequiredKeys<Application> {
let app = body?.application ? body.application : body let app = body?.application ? body.application : body
return { return {
_id: app.appId, _id: app.appId,

2
packages/server/src/api/controllers/public/mapping/index.ts
View File

@ -3,6 +3,7 @@ import applications from "./applications"
import users from "./users" import users from "./users"
import rows from "./rows" import rows from "./rows"
import queries from "./queries" import queries from "./queries"
import views from "./views"
export default { export default {
...tables, ...tables,
@ -10,4 +11,5 @@ export default {
...users, ...users,
...rows, ...rows,
...queries, ...queries,
...views,
} }

3
packages/server/src/api/controllers/public/mapping/queries.ts
View File

@ -1,6 +1,7 @@
import { Query, ExecuteQuery } from "./types" import { Query, ExecuteQuery } from "./types"
import { RequiredKeys } from "@budibase/types"
function query(body: any): Query { function query(body: any): RequiredKeys<Query> {
return { return {
_id: body._id, _id: body._id,
datasourceId: body.datasourceId, datasourceId: body.datasourceId,

3
packages/server/src/api/controllers/public/mapping/rows.ts
View File

@ -1,6 +1,7 @@
import { Row, RowSearch } from "./types" import { Row, RowSearch } from "./types"
import { RequiredKeys } from "@budibase/types"
function row(body: any): Row { function row(body: any): RequiredKeys<Row> {
delete body._rev delete body._rev
// have to input everything, since structure unknown // have to input everything, since structure unknown
return { return {

3
packages/server/src/api/controllers/public/mapping/tables.ts
View File

@ -1,6 +1,7 @@
import { Table } from "./types" import { Table } from "./types"
import { RequiredKeys } from "@budibase/types"
function table(body: any): Table { function table(body: any): RequiredKeys<Table> {
return { return {
_id: body._id, _id: body._id,
name: body.name, name: body.name,

3
packages/server/src/api/controllers/public/mapping/types.ts
View File

@ -9,6 +9,9 @@ export type CreateApplicationParams = components["schemas"]["application"]
export type Table = components["schemas"]["tableOutput"]["data"] export type Table = components["schemas"]["tableOutput"]["data"]
export type CreateTableParams = components["schemas"]["table"] export type CreateTableParams = components["schemas"]["table"]
export type View = components["schemas"]["viewOutput"]["data"]
export type CreateViewParams = components["schemas"]["view"]
export type Row = components["schemas"]["rowOutput"]["data"] export type Row = components["schemas"]["rowOutput"]["data"]
export type RowSearch = components["schemas"]["searchOutput"] export type RowSearch = components["schemas"]["searchOutput"]
export type CreateRowParams = components["schemas"]["row"] export type CreateRowParams = components["schemas"]["row"]

3
packages/server/src/api/controllers/public/mapping/users.ts
View File

@ -1,6 +1,7 @@
import { User } from "./types" import { User } from "./types"
import { RequiredKeys } from "@budibase/types"
function user(body: any): User { function user(body: any): RequiredKeys<User> {
return { return {
_id: body._id, _id: body._id,
email: body.email, email: body.email,

32
packages/server/src/api/controllers/public/mapping/views.ts Normal file
View File

@ -0,0 +1,32 @@
import { View } from "./types"
import { ViewV2, Ctx, RequiredKeys } from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
function view(body: ViewV2): RequiredKeys<View> {
return {
id: body.id,
tableId: body.tableId,
type: body.type,
name: body.name,
schema: body.schema!,
primaryDisplay: body.primaryDisplay,
query: dataFilters.buildQuery(body.query),
sort: body.sort,
}
}
function mapView(ctx: Ctx<{ data: ViewV2 }>): { data: View } {
return {
data: view(ctx.body.data),
}
}
function mapViews(ctx: Ctx<{ data: ViewV2[] }>): { data: View[] } {
const views = ctx.body.data.map((body: ViewV2) => view(body))
return { data: views }
}
export default {
mapView,
mapViews,
}

18
packages/server/src/api/controllers/public/rows.ts
View File

@ -22,13 +22,13 @@ export function fixRow(row: Row, params: any) {
return row return row
} }
export async function search(ctx: UserCtx, next: Next) { function buildSearchRequestBody(ctx: UserCtx) {
let { sort, paginate, bookmark, limit, query } = ctx.request.body let { sort, paginate, bookmark, limit, query } = ctx.request.body
// update the body to the correct format of the internal search // update the body to the correct format of the internal search
if (!sort) { if (!sort) {
sort = {} sort = {}
} }
ctx.request.body = { return {
sort: sort.column, sort: sort.column,
sortType: sort.type, sortType: sort.type,
sortOrder: sort.order, sortOrder: sort.order,
@ -37,10 +37,23 @@ export async function search(ctx: UserCtx, next: Next) {
limit, limit,
query, query,
} }
}
export async function search(ctx: UserCtx, next: Next) {
ctx.request.body = buildSearchRequestBody(ctx)
await rowController.search(ctx) await rowController.search(ctx)
await next() await next()
} }
export async function viewSearch(ctx: UserCtx, next: Next) {
ctx.request.body = buildSearchRequestBody(ctx)
ctx.params = {
viewId: ctx.params.viewId,
}
await rowController.views.searchView(ctx)
await next()
}
export async function create(ctx: UserCtx, next: Next) { export async function create(ctx: UserCtx, next: Next) {
ctx.request.body = fixRow(ctx.request.body, ctx.params) ctx.request.body = fixRow(ctx.request.body, ctx.params)
await rowController.save(ctx) await rowController.save(ctx)
@ -79,4 +92,5 @@ export default {
update, update,
destroy, destroy,
search, search,
viewSearch,
} }

95
packages/server/src/api/controllers/public/views.ts Normal file
View File

@ -0,0 +1,95 @@
import { search as stringSearch } from "./utils"
import * as controller from "../view"
import { ViewV2, UserCtx, UISearchFilter, PublicAPIView } from "@budibase/types"
import { Next } from "koa"
import { merge } from "lodash"
function viewRequest(view: PublicAPIView, params?: { viewId: string }) {
const viewV2: ViewV2 = view
if (!viewV2) {
return viewV2
}
if (params?.viewId) {
viewV2.id = params.viewId
}
if (!view.query) {
viewV2.query = {}
} else {
// public API only has one form of query
viewV2.queryUI = viewV2.query as UISearchFilter
}
viewV2.version = 2
return viewV2
}
function viewResponse(view: ViewV2): PublicAPIView {
// remove our internal structure - always un-necessary
delete view.query
return {
...view,
query: view.queryUI,
}
}
function viewsResponse(views: ViewV2[]): PublicAPIView[] {
return views.map(viewResponse)
}
export async function search(ctx: UserCtx, next: Next) {
const { name } = ctx.request.body
await controller.v2.fetch(ctx)
ctx.body.data = viewsResponse(stringSearch(ctx.body.data, name))
await next()
}
export async function create(ctx: UserCtx, next: Next) {
ctx = merge(ctx, {
request: {
body: viewRequest(ctx.request.body),
},
})
await controller.v2.create(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function read(ctx: UserCtx, next: Next) {
ctx = merge(ctx, {
params: {
viewId: ctx.params.viewId,
},
})
await controller.v2.get(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function update(ctx: UserCtx, next: Next) {
const viewId = ctx.params.viewId
ctx = merge(ctx, {
request: {
body: {
data: viewRequest(ctx.request.body, { viewId }),
},
},
params: {
viewId,
},
})
await controller.v2.update(ctx)
ctx.body.data = viewResponse(ctx.body.data)
await next()
}
export async function destroy(ctx: UserCtx, next: Next) {
await controller.v2.remove(ctx)
await next()
}
export default {
create,
read,
update,
destroy,
search,
}

5
packages/server/src/api/controllers/query/import/sources/curl.ts
View File

@ -4,7 +4,7 @@ import { URL } from "url"
const curlconverter = require("curlconverter") const curlconverter = require("curlconverter")
const parseCurl = (data: string): any => { const parseCurl = (data: string): Promise<any> => {
const curlJson = curlconverter.toJsonString(data) const curlJson = curlconverter.toJsonString(data)
return JSON.parse(curlJson) return JSON.parse(curlJson)
} }
@ -53,8 +53,7 @@ export class Curl extends ImportSource {
isSupported = async (data: string): Promise<boolean> => { isSupported = async (data: string): Promise<boolean> => {
try { try {
const curl = parseCurl(data) this.curl = parseCurl(data)
this.curl = curl
} catch (err) { } catch (err) {
return false return false
} }

3
packages/server/src/api/controllers/query/index.ts
View File

@ -23,6 +23,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core" import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
import { findHBSBlocks } from "@budibase/string-templates" import { findHBSBlocks } from "@budibase/string-templates"
import { ObjectId } from "mongodb"
const Runner = new Thread(ThreadType.QUERY, { const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT, timeoutMs: env.QUERY_THREAD_TIMEOUT,
@ -223,6 +224,8 @@ export async function preview(
} else { } else {
fieldMetadata = makeQuerySchema(FieldType.ARRAY, key) fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
} }
} else if (field instanceof ObjectId) {
fieldMetadata = makeQuerySchema(FieldType.STRING, key)
} else { } else {
fieldMetadata = makeQuerySchema(FieldType.JSON, key) fieldMetadata = makeQuerySchema(FieldType.JSON, key)
} }

1
packages/server/src/api/controllers/row/views.ts
View File

@ -50,6 +50,7 @@ export async function searchView(
result.rows.forEach(r => (r._viewId = view.id)) result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result ctx.body = result
} }
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) { if (request.sort) {
return { return {

7
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -12,6 +12,7 @@ import {
RelationSchemaField, RelationSchemaField,
ViewFieldMetadata, ViewFieldMetadata,
CalculationType, CalculationType,
ViewFetchResponseEnriched,
CountDistinctCalculationFieldMetadata, CountDistinctCalculationFieldMetadata,
CountCalculationFieldMetadata, CountCalculationFieldMetadata,
} from "@budibase/types" } from "@budibase/types"
@ -125,6 +126,12 @@ export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
} }
} }
export async function fetch(ctx: Ctx<void, ViewFetchResponseEnriched>) {
ctx.body = {
data: await sdk.views.getAllEnriched(),
}
}
export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) { export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const view = ctx.request.body const view = ctx.request.body
const { tableId } = view const { tableId } = view

10
packages/server/src/api/routes/public/index.ts
View File

@ -4,19 +4,21 @@ import queryEndpoints from "./queries"
import tableEndpoints from "./tables" import tableEndpoints from "./tables"
import rowEndpoints from "./rows" import rowEndpoints from "./rows"
import userEndpoints from "./users" import userEndpoints from "./users"
import viewEndpoints from "./views"
import roleEndpoints from "./roles" import roleEndpoints from "./roles"
import authorized from "../../../middleware/authorized" import authorized from "../../../middleware/authorized"
import publicApi from "../../../middleware/publicApi" import publicApi from "../../../middleware/publicApi"
import { paramResource, paramSubResource } from "../../../middleware/resourceId" import { paramResource, paramSubResource } from "../../../middleware/resourceId"
import { PermissionType, PermissionLevel } from "@budibase/types" import { PermissionLevel, PermissionType } from "@budibase/types"
import { CtxFn } from "./utils/Endpoint" import { CtxFn } from "./utils/Endpoint"
import mapperMiddleware from "./middleware/mapper" import mapperMiddleware from "./middleware/mapper"
import env from "../../../environment" import env from "../../../environment"
import { middleware, redis } from "@budibase/backend-core"
import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
import cors from "@koa/cors"
// below imports don't have declaration files // below imports don't have declaration files
const Router = require("@koa/router") const Router = require("@koa/router")
const { RateLimit, Stores } = require("koa2-ratelimit") const { RateLimit, Stores } = require("koa2-ratelimit")
import { middleware, redis } from "@budibase/backend-core"
import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
interface KoaRateLimitOptions { interface KoaRateLimitOptions {
socket: { socket: {
@ -81,6 +83,7 @@ const publicRouter = new Router({
if (limiter && !env.isDev()) { if (limiter && !env.isDev()) {
publicRouter.use(limiter) publicRouter.use(limiter)
} }
publicRouter.use(cors())
function addMiddleware( function addMiddleware(
endpoints: any, endpoints: any,
@ -149,6 +152,7 @@ applyAdminRoutes(metricEndpoints)
applyAdminRoutes(roleEndpoints) applyAdminRoutes(roleEndpoints)
applyRoutes(appEndpoints, PermissionType.APP, "appId") applyRoutes(appEndpoints, PermissionType.APP, "appId")
applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId")
applyRoutes(viewEndpoints, PermissionType.VIEW, "viewId")
applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(userEndpoints, PermissionType.USER, "userId")
applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId")
// needs to be applied last for routing purposes, don't override other endpoints // needs to be applied last for routing purposes, don't override other endpoints

30
packages/server/src/api/routes/public/middleware/mapper.ts
View File

@ -1,9 +1,10 @@
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import mapping from "../../../controllers/public/mapping" import mapping from "../../../controllers/public/mapping"
enum Resources { enum Resource {
APPLICATION = "applications", APPLICATION = "applications",
TABLES = "tables", TABLES = "tables",
VIEWS = "views",
ROWS = "rows", ROWS = "rows",
USERS = "users", USERS = "users",
QUERIES = "queries", QUERIES = "queries",
@ -15,7 +16,7 @@ function isAttachment(ctx: Ctx) {
} }
function isArrayResponse(ctx: Ctx) { function isArrayResponse(ctx: Ctx) {
return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body) return ctx.url.endsWith(Resource.SEARCH) || Array.isArray(ctx.body)
} }
function noResponse(ctx: Ctx) { function noResponse(ctx: Ctx) {
@ -38,6 +39,14 @@ function processTables(ctx: Ctx) {
} }
} }
function processViews(ctx: Ctx) {
if (isArrayResponse(ctx)) {
return mapping.mapViews(ctx)
} else {
return mapping.mapView(ctx)
}
}
function processRows(ctx: Ctx) { function processRows(ctx: Ctx) {
if (isArrayResponse(ctx)) { if (isArrayResponse(ctx)) {
return mapping.mapRowSearch(ctx) return mapping.mapRowSearch(ctx)
@ -71,20 +80,27 @@ export default async (ctx: Ctx, next: any) => {
let body = {} let body = {}
switch (urlParts[0]) { switch (urlParts[0]) {
case Resources.APPLICATION: case Resource.APPLICATION:
body = processApplications(ctx) body = processApplications(ctx)
break break
case Resources.TABLES: case Resource.TABLES:
if (urlParts[2] === Resources.ROWS) { if (urlParts[2] === Resource.ROWS) {
body = processRows(ctx) body = processRows(ctx)
} else { } else {
body = processTables(ctx) body = processTables(ctx)
} }
break break
case Resources.USERS: case Resource.VIEWS:
if (urlParts[2] === Resource.ROWS) {
body = processRows(ctx)
} else {
body = processViews(ctx)
}
break
case Resource.USERS:
body = processUsers(ctx) body = processUsers(ctx)
break break
case Resources.QUERIES: case Resource.QUERIES:
body = processQueries(ctx) body = processQueries(ctx)
break break
} }

38
packages/server/src/api/routes/public/rows.ts
View File

@ -1,4 +1,4 @@
import controller from "../../controllers/public/rows" import controller, { viewSearch } from "../../controllers/public/rows"
import Endpoint from "./utils/Endpoint" import Endpoint from "./utils/Endpoint"
import { externalSearchValidator } from "../utils/validators" import { externalSearchValidator } from "../utils/validators"
@ -168,4 +168,40 @@ read.push(
).addMiddleware(externalSearchValidator()) ).addMiddleware(externalSearchValidator())
) )
/**
* @openapi
* /views/{viewId}/rows/search:
* post:
* operationId: rowViewSearch
* summary: Search for rows in a view
* tags:
* - rows
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* requestBody:
* required: true
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/rowSearch'
* responses:
* 200:
* description: The response will contain an array of rows that match the search parameters.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/searchOutput'
* examples:
* search:
* $ref: '#/components/examples/rows'
*/
read.push(
new Endpoint(
"post",
"/views/:viewId/rows/search",
controller.viewSearch
).addMiddleware(externalSearchValidator())
)
export default { read, write } export default { read, write }

118
packages/server/src/api/routes/public/tests/Request.ts
View File

@ -1,13 +1,24 @@
import { User, Table, SearchFilters, Row } from "@budibase/types" import {
User,
Table,
SearchFilters,
Row,
ViewV2Schema,
ViewV2,
ViewV2Type,
PublicAPIView,
} from "@budibase/types"
import { HttpMethod, MakeRequestResponse, generateMakeRequest } from "./utils" import { HttpMethod, MakeRequestResponse, generateMakeRequest } from "./utils"
import TestConfiguration from "../../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
import { Expectations } from "../../../../tests/utilities/api/base"
type RequestOpts = { internal?: boolean; appId?: string } type RequestOpts = { internal?: boolean; appId?: string }
type Response<T> = { data: T }
export interface PublicAPIExpectations { export interface PublicAPIExpectations {
status?: number status?: number
body?: Record<string, any> body?: Record<string, any>
headers?: Record<string, string>
} }
export class PublicAPIRequest { export class PublicAPIRequest {
@ -15,6 +26,7 @@ export class PublicAPIRequest {
private appId: string | undefined private appId: string | undefined
tables: PublicTableAPI tables: PublicTableAPI
views: PublicViewAPI
rows: PublicRowAPI rows: PublicRowAPI
apiKey: string apiKey: string
@ -28,6 +40,7 @@ export class PublicAPIRequest {
this.appId = appId this.appId = appId
this.tables = new PublicTableAPI(this) this.tables = new PublicTableAPI(this)
this.rows = new PublicRowAPI(this) this.rows = new PublicRowAPI(this)
this.views = new PublicViewAPI(this)
} }
static async init(config: TestConfiguration, user: User, opts?: RequestOpts) { static async init(config: TestConfiguration, user: User, opts?: RequestOpts) {
@ -59,6 +72,12 @@ export class PublicAPIRequest {
if (expectations?.body) { if (expectations?.body) {
expect(res.body).toEqual(expectations?.body) expect(res.body).toEqual(expectations?.body)
} }
if (expectations?.headers) {
for (let [header, value] of Object.entries(expectations.headers)) {
const found = res.headers[header]
expect(found?.toLowerCase()).toEqual(value)
}
}
return res.body return res.body
} }
} }
@ -73,9 +92,16 @@ export class PublicTableAPI {
async create( async create(
table: Table, table: Table,
expectations?: PublicAPIExpectations expectations?: PublicAPIExpectations
): Promise<{ data: Table }> { ): Promise<Response<Table>> {
return this.request.send("post", "/tables", table, expectations) return this.request.send("post", "/tables", table, expectations)
} }
async search(
name: string,
expectations?: PublicAPIExpectations
): Promise<Response<Table[]>> {
return this.request.send("post", "/tables/search", { name }, expectations)
}
} }
export class PublicRowAPI { export class PublicRowAPI {
@ -85,11 +111,24 @@ export class PublicRowAPI {
this.request = request this.request = request
} }
async create(
tableId: string,
row: Row,
expectations?: PublicAPIExpectations
): Promise<Response<Row>> {
return this.request.send(
"post",
`/tables/${tableId}/rows`,
row,
expectations
)
}
async search( async search(
tableId: string, tableId: string,
query: SearchFilters, query: SearchFilters,
expectations?: PublicAPIExpectations expectations?: PublicAPIExpectations
): Promise<{ data: Row[] }> { ): Promise<Response<Row[]>> {
return this.request.send( return this.request.send(
"post", "post",
`/tables/${tableId}/rows/search`, `/tables/${tableId}/rows/search`,
@ -99,4 +138,75 @@ export class PublicRowAPI {
expectations expectations
) )
} }
async viewSearch(
viewId: string,
query: SearchFilters,
expectations?: PublicAPIExpectations
): Promise<Response<Row[]>> {
return this.request.send(
"post",
`/views/${viewId}/rows/search`,
{
query,
},
expectations
)
}
}
export class PublicViewAPI {
request: PublicAPIRequest
constructor(request: PublicAPIRequest) {
this.request = request
}
async create(
view: Omit<PublicAPIView, "id" | "version">,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("post", "/views", view, expectations)
}
async update(
viewId: string,
view: Omit<PublicAPIView, "id" | "version">,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("put", `/views/${viewId}`, view, expectations)
}
async destroy(
viewId: string,
expectations?: PublicAPIExpectations
): Promise<void> {
return this.request.send(
"delete",
`/views/${viewId}`,
undefined,
expectations
)
}
async find(
viewId: string,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView>> {
return this.request.send("get", `/views/${viewId}`, undefined, expectations)
}
async search(
viewName: string,
expectations?: PublicAPIExpectations
): Promise<Response<PublicAPIView[]>> {
return this.request.send(
"post",
"/views/search",
{
name: viewName,
},
expectations
)
}
} }

21
packages/server/src/api/routes/public/tests/cors.spec.ts Normal file
View File

@ -0,0 +1,21 @@
import * as setup from "../../tests/utilities"
import { PublicAPIRequest } from "./Request"
describe("check public API security", () => {
const config = setup.getConfig()
let request: PublicAPIRequest
beforeAll(async () => {
await config.init()
request = await PublicAPIRequest.init(config, await config.globalUser())
})
it("should have Access-Control-Allow-Origin set to *", async () => {
await request.tables.search("", {
status: 200,
headers: {
"access-control-allow-origin": "*",
},
})
})
})

95
packages/server/src/api/routes/public/tests/views.spec.ts Normal file
View File

@ -0,0 +1,95 @@
import * as setup from "../../tests/utilities"
import { basicTable } from "../../../../tests/utilities/structures"
import { BasicOperator, Table, UILogicalOperator } from "@budibase/types"
import { PublicAPIRequest } from "./Request"
import { generator } from "@budibase/backend-core/tests"
describe("check public API security", () => {
const config = setup.getConfig()
let request: PublicAPIRequest, table: Table
beforeAll(async () => {
await config.init()
request = await PublicAPIRequest.init(config, await config.globalUser())
table = (await request.tables.create(basicTable())).data
})
function baseView() {
return {
name: generator.word(),
tableId: table._id!,
query: {},
schema: {
name: {
readonly: true,
visible: true,
},
},
}
}
it("should be able to create a view", async () => {
await request.views.create(baseView(), { status: 201 })
})
it("should be able to update a view", async () => {
const view = await request.views.create(baseView(), { status: 201 })
const response = await request.views.update(view.data.id, {
...view.data,
name: "new name",
})
})
it("should be able to search views", async () => {
const viewName = "view to search for"
const view = await request.views.create(
{
...baseView(),
name: viewName,
},
{ status: 201 }
)
const results = await request.views.search(viewName, {
status: 200,
})
expect(results.data.length).toEqual(1)
expect(results.data[0].id).toEqual(view.data.id)
})
it("should be able to delete a view", async () => {
const view = await request.views.create(baseView(), { status: 201 })
const result = await request.views.destroy(view.data.id, { status: 204 })
expect(result).toBeDefined()
})
it("should be able to search rows through a view", async () => {
const row1 = await request.rows.create(
table._id!,
{ name: "hello world" },
{ status: 200 }
)
await request.rows.create(table._id!, { name: "foo bar" }, { status: 200 })
const response = await request.views.create(
{
...baseView(),
query: {
logicalOperator: UILogicalOperator.ANY,
groups: [
{
filters: [
{
operator: BasicOperator.STRING,
field: "name",
value: "hello",
},
],
},
],
},
},
{ status: 201 }
)
const results = await request.rows.viewSearch(response.data.id, {})
expect(results.data.length).toEqual(1)
})
})

165
packages/server/src/api/routes/public/views.ts Normal file
View File

@ -0,0 +1,165 @@
import controller from "../../controllers/public/views"
import Endpoint from "./utils/Endpoint"
import { viewValidator, nameValidator } from "../utils/validators"
const read = [],
write = []
/**
* @openapi
* /views:
* post:
* operationId: viewCreate
* summary: Create a view
* description: Create a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/appId'
* requestBody:
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/view'
* examples:
* view:
* $ref: '#/components/examples/view'
* responses:
* 200:
* description: Returns the created view, including the ID which has been generated for it.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(
new Endpoint("post", "/views", controller.create).addMiddleware(
viewValidator()
)
)
/**
* @openapi
* /views/{viewId}:
* put:
* operationId: viewUpdate
* summary: Update a view
* description: Update a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* requestBody:
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/view'
* examples:
* view:
* $ref: '#/components/examples/view'
* responses:
* 200:
* description: Returns the updated view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(
new Endpoint("put", "/views/:viewId", controller.update).addMiddleware(
viewValidator()
)
)
/**
* @openapi
* /views/{viewId}:
* delete:
* operationId: viewDestroy
* summary: Delete a view
* description: Delete a view, this can be against an internal or external table.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* responses:
* 200:
* description: Returns the deleted view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
write.push(new Endpoint("delete", "/views/:viewId", controller.destroy))
/**
* @openapi
* /views/{viewId}:
* get:
* operationId: viewGetById
* summary: Retrieve a view
* description: Lookup a view, this could be internal or external.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/viewId'
* - $ref: '#/components/parameters/appId'
* responses:
* 200:
* description: Returns the retrieved view.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewOutput'
* examples:
* view:
* $ref: '#/components/examples/view'
*/
read.push(new Endpoint("get", "/views/:viewId", controller.read))
/**
* @openapi
* /views/search:
* post:
* operationId: viewSearch
* summary: Search for views
* description: Based on view properties (currently only name) search for views.
* tags:
* - views
* parameters:
* - $ref: '#/components/parameters/appId'
* requestBody:
* required: true
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/nameSearch'
* responses:
* 200:
* description: Returns the found views, based on the search parameters.
* content:
* application/json:
* schema:
* $ref: '#/components/schemas/viewSearch'
* examples:
* views:
* $ref: '#/components/examples/views'
*/
read.push(
new Endpoint("post", "/views/search", controller.search).addMiddleware(
nameValidator()
)
)
export default { read, write }

13
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -164,9 +164,12 @@ describe("/datasources", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({
{ name: "%s", exclude: [DatabaseName.MONGODB, DatabaseName.SQS] }, exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
({ config, dsProvider }) => { })
if (descriptions.length) {
describe.each(descriptions)("$dbName", ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let rawDatasource: Datasource let rawDatasource: Datasource
let client: Knex let client: Knex
@ -492,5 +495,5 @@ datasourceDescribe(
) )
}) })
}) })
} })
) }

1692
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

File diff suppressed because it is too large Load Diff

1252
packages/server/src/api/routes/tests/queries/mongodb.spec.ts
View File

File diff suppressed because it is too large Load Diff

6199
packages/server/src/api/routes/tests/row.spec.ts
View File

File diff suppressed because it is too large Load Diff

110
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -977,63 +977,69 @@ describe("/rowsActions", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({
{ name: "row actions (%s)", only: [DatabaseName.SQS, DatabaseName.POSTGRES] }, only: [DatabaseName.SQS, DatabaseName.POSTGRES],
({ config, dsProvider, isInternal }) => { })
let datasource: Datasource | undefined
beforeAll(async () => { if (descriptions.length) {
const ds = await dsProvider() describe.each(descriptions)(
datasource = ds.datasource "row actions ($dbName)",
}) ({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
async function getTable(): Promise<Table> { beforeAll(async () => {
if (isInternal) { const ds = await dsProvider()
await config.api.application.addSampleData(config.getAppId()) datasource = ds.datasource
const tables = await config.api.table.fetch() })
return tables.find(t => t.sourceId === DEFAULT_BB_DATASOURCE_ID)!
} else {
const table = await config.api.table.save(
setup.structures.tableForDatasource(datasource!)
)
return table
}
}
it("should delete all the row actions (and automations) for its tables when a datasource is deleted", async () => { async function getTable(): Promise<Table> {
async function getRowActionsFromDb(tableId: string) { if (isInternal) {
return await context.doInAppContext(config.getAppId(), async () => { await config.api.application.addSampleData(config.getAppId())
const db = context.getAppDB() const tables = await config.api.table.fetch()
const tableDoc = await db.tryGet<TableRowActions>( return tables.find(t => t.sourceId === DEFAULT_BB_DATASOURCE_ID)!
generateRowActionsID(tableId) } else {
const table = await config.api.table.save(
setup.structures.tableForDatasource(datasource!)
) )
return tableDoc return table
}) }
} }
const table = await getTable() it("should delete all the row actions (and automations) for its tables when a datasource is deleted", async () => {
const tableId = table._id! async function getRowActionsFromDb(tableId: string) {
return await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB()
const tableDoc = await db.tryGet<TableRowActions>(
generateRowActionsID(tableId)
)
return tableDoc
})
}
await config.api.rowAction.save(tableId, { const table = await getTable()
name: generator.guid(), const tableId = table._id!
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
await config.api.rowAction.save(tableId, {
name: generator.guid(),
})
const { actions } = (await getRowActionsFromDb(tableId))!
expect(Object.entries(actions)).toHaveLength(2)
const { automations } = await config.api.automation.fetch()
expect(automations).toHaveLength(2)
const datasource = await config.api.datasource.get(table.sourceId)
await config.api.datasource.delete(datasource)
const automationsResp = await config.api.automation.fetch()
expect(automationsResp.automations).toHaveLength(0)
expect(await getRowActionsFromDb(tableId)).toBeUndefined()
}) })
await config.api.rowAction.save(tableId, { }
name: generator.guid(), )
}) }
const { actions } = (await getRowActionsFromDb(tableId))!
expect(Object.entries(actions)).toHaveLength(2)
const { automations } = await config.api.automation.fetch()
expect(automations).toHaveLength(2)
const datasource = await config.api.datasource.get(table.sourceId)
await config.api.datasource.delete(datasource)
const automationsResp = await config.api.automation.fetch()
expect(automationsResp.automations).toHaveLength(0)
expect(await getRowActionsFromDb(tableId)).toBeUndefined()
})
}
)

7201
packages/server/src/api/routes/tests/search.spec.ts
View File

File diff suppressed because it is too large Load Diff

2672
packages/server/src/api/routes/tests/table.spec.ts
View File

File diff suppressed because it is too large Load Diff

8205
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

File diff suppressed because it is too large Load Diff

79
packages/server/src/api/routes/utils/validators.ts
View File

@ -9,6 +9,13 @@ import {
Table, Table,
WebhookActionType, WebhookActionType,
BuiltinPermissionID, BuiltinPermissionID,
ViewV2Type,
SortOrder,
SortType,
UILogicalOperator,
BasicOperator,
ArrayOperator,
RangeOperator,
} from "@budibase/types" } from "@budibase/types"
import Joi, { CustomValidator } from "joi" import Joi, { CustomValidator } from "joi"
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core" import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
@ -66,6 +73,66 @@ export function tableValidator() {
) )
} }
function searchUIFilterValidator() {
const logicalOperator = Joi.string().valid(
...Object.values(UILogicalOperator)
)
const operators = [
...Object.values(BasicOperator),
...Object.values(ArrayOperator),
...Object.values(RangeOperator),
]
const filters = Joi.array().items(
Joi.object({
operator: Joi.string()
.valid(...operators)
.required(),
field: Joi.string().required(),
// could do with better validation of value based on operator
value: Joi.any().required(),
})
)
return Joi.object({
logicalOperator,
onEmptyFilter: Joi.string().valid(...Object.values(EmptyFilterOption)),
groups: Joi.array().items(
Joi.object({
logicalOperator,
filters,
groups: Joi.array().items(
Joi.object({
filters,
logicalOperator,
})
),
})
),
})
}
export function viewValidator() {
return auth.joiValidator.body(
Joi.object({
id: OPTIONAL_STRING,
tableId: Joi.string().required(),
name: Joi.string().required(),
type: Joi.string().optional().valid(null, ViewV2Type.CALCULATION),
primaryDisplay: OPTIONAL_STRING,
schema: Joi.object().required(),
query: searchUIFilterValidator().optional(),
sort: Joi.object({
field: Joi.string().required(),
order: Joi.string()
.optional()
.valid(...Object.values(SortOrder)),
type: Joi.string()
.optional()
.valid(...Object.values(SortType)),
}).optional(),
})
)
}
export function nameValidator() { export function nameValidator() {
return auth.joiValidator.body( return auth.joiValidator.body(
Joi.object({ Joi.object({
@ -91,8 +158,7 @@ export function datasourceValidator() {
) )
} }
function filterObject(opts?: { unknown: boolean }) { function searchFiltersValidator() {
const { unknown = true } = opts || {}
const conditionalFilteringObject = () => const conditionalFilteringObject = () =>
Joi.object({ Joi.object({
conditions: Joi.array().items(Joi.link("#schema")).required(), conditions: Joi.array().items(Joi.link("#schema")).required(),
@ -119,7 +185,14 @@ function filterObject(opts?: { unknown: boolean }) {
fuzzyOr: Joi.forbidden(), fuzzyOr: Joi.forbidden(),
documentType: Joi.forbidden(), documentType: Joi.forbidden(),
} }
return Joi.object(filtersValidators).unknown(unknown).id("schema")
return Joi.object(filtersValidators)
}
function filterObject(opts?: { unknown: boolean }) {
const { unknown = true } = opts || {}
return searchFiltersValidator().unknown(unknown).id("schema")
} }
export function internalSearchValidator() { export function internalSearchValidator() {

5
packages/server/src/api/routes/view.ts
View File

@ -8,6 +8,11 @@ import { permissions } from "@budibase/backend-core"
const router: Router = new Router() const router: Router = new Router()
router router
.get(
"/api/v2/views",
authorized(permissions.BUILDER),
viewController.v2.fetch
)
.get( .get(
"/api/v2/views/:viewId", "/api/v2/views/:viewId",
authorizedResource( authorizedResource(

3
packages/server/src/automations/steps/serverLog.ts
View File

@ -61,6 +61,9 @@ export async function run({
inputs: ServerLogStepInputs inputs: ServerLogStepInputs
appId: string appId: string
}): Promise<ServerLogStepOutputs> { }): Promise<ServerLogStepOutputs> {
if (typeof inputs.text !== "string") {
inputs.text = JSON.stringify(inputs.text)
}
const message = `App ${appId} - ${inputs.text}` const message = `App ${appId} - ${inputs.text}`
console.log(message) console.log(message)
return { return {

125
packages/server/src/automations/tests/executeQuery.spec.ts
View File

@ -7,71 +7,74 @@ import {
import { Knex } from "knex" import { Knex } from "knex"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
datasourceDescribe( const descriptions = datasourceDescribe({
{ exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
name: "execute query action", })
exclude: [DatabaseName.MONGODB, DatabaseName.SQS],
},
({ config, dsProvider }) => {
let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
beforeAll(async () => { if (descriptions.length) {
const ds = await dsProvider() describe.each(descriptions)(
datasource = ds.datasource! "execute query action ($dbName)",
client = ds.client! ({ config, dsProvider }) => {
}) let tableName: string
let client: Knex
let datasource: Datasource
let query: Query
beforeEach(async () => { beforeAll(async () => {
tableName = generator.guid() const ds = await dsProvider()
await client.schema.createTable(tableName, table => { datasource = ds.datasource!
table.string("a") client = ds.client!
table.integer("b")
}) })
await client(tableName).insert({ a: "string", b: 1 })
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
afterEach(async () => { beforeEach(async () => {
await client.schema.dropTable(tableName) tableName = generator.guid()
}) await client.schema.createTable(tableName, table => {
table.string("a")
table.integer("b")
})
await client(tableName).insert({ a: "string", b: 1 })
query = await setup.saveTestQuery(config, client, tableName, datasource)
})
it("should be able to execute a query", async () => { afterEach(async () => {
let res = await setup.runStep( await client.schema.dropTable(tableName)
config, })
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: query._id },
}
)
expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(true)
})
it("should handle a null query value", async () => { it("should be able to execute a query", async () => {
let res = await setup.runStep( let res = await setup.runStep(
config, config,
setup.actions.EXECUTE_QUERY.stepId, setup.actions.EXECUTE_QUERY.stepId,
{ {
query: null, query: { queryId: query._id },
} }
) )
expect(res.response.message).toEqual("Invalid inputs") expect(res.response).toEqual([{ a: "string", b: 1 }])
expect(res.success).toEqual(false) expect(res.success).toEqual(true)
}) })
it("should handle an error executing a query", async () => { it("should handle a null query value", async () => {
let res = await setup.runStep( let res = await setup.runStep(
config, config,
setup.actions.EXECUTE_QUERY.stepId, setup.actions.EXECUTE_QUERY.stepId,
{ {
query: { queryId: "wrong_id" }, query: null,
} }
) )
expect(res.response).toBeDefined() expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false) expect(res.success).toEqual(false)
}) })
}
) it("should handle an error executing a query", async () => {
let res = await setup.runStep(
config,
setup.actions.EXECUTE_QUERY.stepId,
{
query: { queryId: "wrong_id" },
}
)
expect(res.response).toBeDefined()
expect(res.success).toEqual(false)
})
}
)
}

139
packages/server/src/automations/tests/executeScript.spec.ts
View File

@ -1,50 +1,123 @@
import { getConfig, afterAll as _afterAll, runStep, actions } from "./utilities" import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
import * as automation from "../index"
import * as setup from "./utilities"
import { Table } from "@budibase/types"
describe("test the execute script action", () => { describe("Execute Script Automations", () => {
let config = getConfig() let config = setup.getConfig(),
table: Table
beforeAll(async () => { beforeEach(async () => {
await automation.init()
await config.init() await config.init()
table = await config.createTable()
await config.createRow()
}) })
afterAll(_afterAll)
it("should be able to execute a script", async () => { afterAll(setup.afterAll)
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, {
code: "return 1 + 1", it("should execute a basic script and return the result", async () => {
const builder = createAutomationBuilder({
name: "Basic Script Execution",
}) })
expect(res.value).toEqual(2)
expect(res.success).toEqual(true) const results = await builder
.appAction({ fields: {} })
.executeScript({ code: "return 2 + 2" })
.run()
expect(results.steps[0].outputs.value).toEqual(4)
}) })
it("should handle a null value", async () => { it("should access bindings from previous steps", async () => {
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, { const builder = createAutomationBuilder({
code: null, name: "Access Bindings",
}) })
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false) const results = await builder
.appAction({ fields: { data: [1, 2, 3] } })
.executeScript(
{
code: "return trigger.fields.data.map(x => x * 2)",
},
{ stepId: "binding-script-step" }
)
.run()
expect(results.steps[0].outputs.value).toEqual([2, 4, 6])
}) })
it("should be able to get a value from context", async () => { it("should handle script execution errors gracefully", async () => {
const res = await runStep( const builder = createAutomationBuilder({
config, name: "Handle Script Errors",
actions.EXECUTE_SCRIPT.stepId, })
{
code: "return steps.map(d => d.value)", const results = await builder
}, .appAction({ fields: {} })
{ .executeScript({ code: "return nonexistentVariable.map(x => x)" })
steps: [{ value: 0 }, { value: 1 }], .run()
}
expect(results.steps[0].outputs.response).toContain(
"ReferenceError: nonexistentVariable is not defined"
) )
expect(res.value).toEqual([0, 1]) expect(results.steps[0].outputs.success).toEqual(false)
expect(res.response).toBeUndefined()
expect(res.success).toEqual(true)
}) })
it("should be able to handle an error gracefully", async () => { it("should handle conditional logic in scripts", async () => {
const res = await runStep(config, actions.EXECUTE_SCRIPT.stepId, { const builder = createAutomationBuilder({
code: "return something.map(x => x.name)", name: "Conditional Script Logic",
}) })
expect(res.response).toEqual("ReferenceError: something is not defined")
expect(res.success).toEqual(false) const results = await builder
.appAction({ fields: { value: 10 } })
.executeScript({
code: `
if (trigger.fields.value > 5) {
return "Value is greater than 5";
} else {
return "Value is 5 or less";
}
`,
})
.run()
expect(results.steps[0].outputs.value).toEqual("Value is greater than 5")
})
it("should use multiple steps and validate script execution", async () => {
const builder = createAutomationBuilder({
name: "Multi-Step Script Execution",
})
const results = await builder
.appAction({ fields: {} })
.serverLog(
{ text: "Starting multi-step automation" },
{ stepId: "start-log-step" }
)
.createRow(
{ row: { name: "Test Row", value: 42, tableId: table._id } },
{ stepId: "abc123" }
)
.executeScript(
{
code: `
const createdRow = steps['abc123'];
return createdRow.row.value * 2;
`,
},
{ stepId: "ScriptingStep1" }
)
.serverLog({
text: `Final result is {{ steps.ScriptingStep1.value }}`,
})
.run()
expect(results.steps[0].outputs.message).toContain(
"Starting multi-step automation"
)
expect(results.steps[1].outputs.row.value).toEqual(42)
expect(results.steps[2].outputs.value).toEqual(84)
expect(results.steps[3].outputs.message).toContain("Final result is 84")
}) })
}) })

11
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -433,9 +433,10 @@ describe("Automation Scenarios", () => {
}) })
}) })
datasourceDescribe( const descriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
{ name: "", only: [DatabaseName.MYSQL] },
({ config, dsProvider }) => { if (descriptions.length) {
describe.each(descriptions)("/rows ($dbName)", ({ config, dsProvider }) => {
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -531,5 +532,5 @@ datasourceDescribe(
) )
}) })
}) })
} })
) }

13
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View File

@ -34,6 +34,7 @@ import {
SearchFilters, SearchFilters,
Branch, Branch,
FilterStepInputs, FilterStepInputs,
ExecuteScriptStepInputs,
} from "@budibase/types" } from "@budibase/types"
import TestConfiguration from "../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import * as setup from "../utilities" import * as setup from "../utilities"
@ -201,6 +202,18 @@ class BaseStepBuilder {
) )
} }
executeScript(
input: ExecuteScriptStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.EXECUTE_SCRIPT,
BUILTIN_ACTION_DEFINITIONS.EXECUTE_SCRIPT,
input,
opts
)
}
filter(input: FilterStepInputs): this { filter(input: FilterStepInputs): this {
return this.step( return this.step(
AutomationActionStepId.FILTER, AutomationActionStepId.FILTER,

588
packages/server/src/definitions/openapi.ts
View File

@ -65,6 +65,9 @@ export interface paths {
"/tables/{tableId}/rows/search": { "/tables/{tableId}/rows/search": {
post: operations["rowSearch"]; post: operations["rowSearch"];
}; };
"/views/{viewId}/rows/search": {
post: operations["rowViewSearch"];
};
"/tables": { "/tables": {
/** Create a table, this could be internal or external. */ /** Create a table, this could be internal or external. */
post: operations["tableCreate"]; post: operations["tableCreate"];
@ -93,6 +96,22 @@ export interface paths {
/** Based on user properties (currently only name) search for users. */ /** Based on user properties (currently only name) search for users. */
post: operations["userSearch"]; post: operations["userSearch"];
}; };
"/views": {
/** Create a view, this can be against an internal or external table. */
post: operations["viewCreate"];
};
"/views/{viewId}": {
/** Lookup a view, this could be internal or external. */
get: operations["viewGetById"];
/** Update a view, this can be against an internal or external table. */
put: operations["viewUpdate"];
/** Delete a view, this can be against an internal or external table. */
delete: operations["viewDestroy"];
};
"/views/search": {
/** Based on view properties (currently only name) search for views. */
post: operations["viewSearch"];
};
} }
export interface components { export interface components {
@ -813,10 +832,442 @@ export interface components {
userIds: string[]; userIds: string[];
}; };
}; };
/** @description The view to be created/updated. */
view: {
/** @description The name of the view. */
name: string;
/** @description The ID of the table this view is based on. */
tableId: string;
/**
* @description The type of view - standard (empty value) or calculation.
* @enum {string}
*/
type?: "calculation";
/** @description A column used to display rows from this view - usually used when rendered in tables. */
primaryDisplay?: string;
/** @description Search parameters for view */
query?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/**
* @description If no filters match, should the view return all rows, or no rows.
* @enum {string}
*/
onEmptyFilter?: "all" | "none";
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
}[];
}[];
};
sort?: {
/** @description The field from the table/view schema to sort on. */
field: string;
/**
* @description The order in which to sort.
* @enum {string}
*/
order?: "ascending" | "descending";
/**
* @description The type of sort to perform (by number, or by alphabetically).
* @enum {string}
*/
type?: "string" | "number";
};
schema: {
[key: string]:
| {
/** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */
visible?: boolean;
/** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */
readonly?: boolean;
/** @description A number defining where the column shows up in tables, lowest being first. */
order?: number;
/** @description A width for the column, defined in pixels - this affects rendering in tables. */
width?: number;
/** @description If this is a relationship column, we can set the columns we wish to include */
column?: {
readonly?: boolean;
}[];
}
| {
/**
* @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.
* @enum {string}
*/
calculationType?: "sum" | "avg" | "count" | "min" | "max";
/** @description The field from the table to perform the calculation on. */
field?: string;
/** @description Can be used in tandem with the count calculation type, to count unique entries. */
distinct?: boolean;
};
};
};
viewOutput: {
/** @description The view to be created/updated. */
data: {
/** @description The name of the view. */
name: string;
/** @description The ID of the table this view is based on. */
tableId: string;
/**
* @description The type of view - standard (empty value) or calculation.
* @enum {string}
*/
type?: "calculation";
/** @description A column used to display rows from this view - usually used when rendered in tables. */
primaryDisplay?: string;
/** @description Search parameters for view */
query?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/**
* @description If no filters match, should the view return all rows, or no rows.
* @enum {string}
*/
onEmptyFilter?: "all" | "none";
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
}[];
}[];
};
sort?: {
/** @description The field from the table/view schema to sort on. */
field: string;
/**
* @description The order in which to sort.
* @enum {string}
*/
order?: "ascending" | "descending";
/**
* @description The type of sort to perform (by number, or by alphabetically).
* @enum {string}
*/
type?: "string" | "number";
};
schema: {
[key: string]:
| {
/** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */
visible?: boolean;
/** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */
readonly?: boolean;
/** @description A number defining where the column shows up in tables, lowest being first. */
order?: number;
/** @description A width for the column, defined in pixels - this affects rendering in tables. */
width?: number;
/** @description If this is a relationship column, we can set the columns we wish to include */
column?: {
readonly?: boolean;
}[];
}
| {
/**
* @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.
* @enum {string}
*/
calculationType?: "sum" | "avg" | "count" | "min" | "max";
/** @description The field from the table to perform the calculation on. */
field?: string;
/** @description Can be used in tandem with the count calculation type, to count unique entries. */
distinct?: boolean;
};
};
/** @description The ID of the view. */
id: string;
};
};
viewSearch: {
data: {
/** @description The name of the view. */
name: string;
/** @description The ID of the table this view is based on. */
tableId: string;
/**
* @description The type of view - standard (empty value) or calculation.
* @enum {string}
*/
type?: "calculation";
/** @description A column used to display rows from this view - usually used when rendered in tables. */
primaryDisplay?: string;
/** @description Search parameters for view */
query?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/**
* @description If no filters match, should the view return all rows, or no rows.
* @enum {string}
*/
onEmptyFilter?: "all" | "none";
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
/** @description A grouping of filters to be applied. */
groups?: {
/**
* @description When using groups this defines whether all of the filters must match, or only one of them.
* @enum {string}
*/
logicalOperator?: "all" | "any";
/** @description A list of filters to apply */
filters?: {
/**
* @description The type of search operation which is being performed.
* @enum {string}
*/
operator?:
| "equal"
| "notEqual"
| "empty"
| "notEmpty"
| "fuzzy"
| "string"
| "contains"
| "notContains"
| "containsAny"
| "oneOf"
| "range";
/** @description The field in the view to perform the search on. */
field?: string;
/** @description The value to search for - the type will depend on the operator in use. */
value?:
| string
| number
| boolean
| { [key: string]: unknown }
| unknown[];
}[];
}[];
}[];
};
sort?: {
/** @description The field from the table/view schema to sort on. */
field: string;
/**
* @description The order in which to sort.
* @enum {string}
*/
order?: "ascending" | "descending";
/**
* @description The type of sort to perform (by number, or by alphabetically).
* @enum {string}
*/
type?: "string" | "number";
};
schema: {
[key: string]:
| {
/** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */
visible?: boolean;
/** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */
readonly?: boolean;
/** @description A number defining where the column shows up in tables, lowest being first. */
order?: number;
/** @description A width for the column, defined in pixels - this affects rendering in tables. */
width?: number;
/** @description If this is a relationship column, we can set the columns we wish to include */
column?: {
readonly?: boolean;
}[];
}
| {
/**
* @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.
* @enum {string}
*/
calculationType?: "sum" | "avg" | "count" | "min" | "max";
/** @description The field from the table to perform the calculation on. */
field?: string;
/** @description Can be used in tandem with the count calculation type, to count unique entries. */
distinct?: boolean;
};
};
/** @description The ID of the view. */
id: string;
}[];
};
}; };
parameters: { parameters: {
/** @description The ID of the table which this request is targeting. */ /** @description The ID of the table which this request is targeting. */
tableId: string; tableId: string;
/** @description The ID of the view which this request is targeting. */
viewId: string;
/** @description The ID of the row which this request is targeting. */ /** @description The ID of the row which this request is targeting. */
rowId: string; rowId: string;
/** @description The ID of the app which this request is targeting. */ /** @description The ID of the app which this request is targeting. */
@ -1213,6 +1664,31 @@ export interface operations {
}; };
}; };
}; };
rowViewSearch: {
parameters: {
path: {
/** The ID of the view which this request is targeting. */
viewId: components["parameters"]["viewId"];
};
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** The response will contain an array of rows that match the search parameters. */
200: {
content: {
"application/json": components["schemas"]["searchOutput"];
};
};
};
requestBody: {
content: {
"application/json": components["schemas"]["rowSearch"];
};
};
};
/** Create a table, this could be internal or external. */ /** Create a table, this could be internal or external. */
tableCreate: { tableCreate: {
parameters: { parameters: {
@ -1409,6 +1885,118 @@ export interface operations {
}; };
}; };
}; };
/** Create a view, this can be against an internal or external table. */
viewCreate: {
parameters: {
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** Returns the created view, including the ID which has been generated for it. */
200: {
content: {
"application/json": components["schemas"]["viewOutput"];
};
};
};
requestBody: {
content: {
"application/json": components["schemas"]["view"];
};
};
};
/** Lookup a view, this could be internal or external. */
viewGetById: {
parameters: {
path: {
/** The ID of the view which this request is targeting. */
viewId: components["parameters"]["viewId"];
};
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** Returns the retrieved view. */
200: {
content: {
"application/json": components["schemas"]["viewOutput"];
};
};
};
};
/** Update a view, this can be against an internal or external table. */
viewUpdate: {
parameters: {
path: {
/** The ID of the view which this request is targeting. */
viewId: components["parameters"]["viewId"];
};
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** Returns the updated view. */
200: {
content: {
"application/json": components["schemas"]["viewOutput"];
};
};
};
requestBody: {
content: {
"application/json": components["schemas"]["view"];
};
};
};
/** Delete a view, this can be against an internal or external table. */
viewDestroy: {
parameters: {
path: {
/** The ID of the view which this request is targeting. */
viewId: components["parameters"]["viewId"];
};
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** Returns the deleted view. */
200: {
content: {
"application/json": components["schemas"]["viewOutput"];
};
};
};
};
/** Based on view properties (currently only name) search for views. */
viewSearch: {
parameters: {
header: {
/** The ID of the app which this request is targeting. */
"x-budibase-app-id": components["parameters"]["appId"];
};
};
responses: {
/** Returns the found views, based on the search parameters. */
200: {
content: {
"application/json": components["schemas"]["viewSearch"];
};
};
};
requestBody: {
content: {
"application/json": components["schemas"]["nameSearch"];
};
};
};
} }
export interface external {} export interface external {}

204
packages/server/src/integration-test/mysql.spec.ts
View File

@ -10,119 +10,123 @@ function uniqueTableName(length?: number): string {
.substring(0, length || 10) .substring(0, length || 10)
} }
datasourceDescribe( const mainDescriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
{
name: "Integration compatibility with mysql search_path",
only: [DatabaseName.MYSQL],
},
({ config, dsProvider }) => {
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
const database = generator.guid() if (mainDescriptions.length) {
const database2 = generator.guid() describe.each(mainDescriptions)(
"/Integration compatibility with mysql search_path ($dbName)",
({ config, dsProvider }) => {
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
beforeAll(async () => { const database = generator.guid()
const ds = await dsProvider() const database2 = generator.guid()
rawDatasource = ds.rawDatasource!
datasource = ds.datasource!
client = ds.client!
await client.raw(`CREATE DATABASE \`${database}\`;`) beforeAll(async () => {
await client.raw(`CREATE DATABASE \`${database2}\`;`) const ds = await dsProvider()
rawDatasource = ds.rawDatasource!
datasource = ds.datasource!
client = ds.client!
rawDatasource.config!.database = database await client.raw(`CREATE DATABASE \`${database}\`;`)
datasource = await config.api.datasource.create(rawDatasource) await client.raw(`CREATE DATABASE \`${database2}\`;`)
})
afterAll(async () => { rawDatasource.config!.database = database
await client.raw(`DROP DATABASE \`${database}\`;`) datasource = await config.api.datasource.create(rawDatasource)
await client.raw(`DROP DATABASE \`${database2}\`;`)
})
it("discovers tables from any schema in search path", async () => {
await client.schema.createTable(`${database}.table1`, table => {
table.increments("id1").primary()
}) })
const res = await config.api.datasource.info(datasource)
expect(res.tableNames).toBeDefined()
expect(res.tableNames).toEqual(expect.arrayContaining(["table1"]))
})
it("does not mix columns from different tables", async () => { afterAll(async () => {
const repeated_table_name = "table_same_name" await client.raw(`DROP DATABASE \`${database}\`;`)
await client.schema.createTable( await client.raw(`DROP DATABASE \`${database2}\`;`)
`${database}.${repeated_table_name}`,
table => {
table.increments("id").primary()
table.string("val1")
}
)
await client.schema.createTable(
`${database2}.${repeated_table_name}`,
table => {
table.increments("id2").primary()
table.string("val2")
}
)
const res = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
tablesFilter: [repeated_table_name],
}) })
expect(res.datasource.entities![repeated_table_name].schema).toBeDefined()
const schema = res.datasource.entities![repeated_table_name].schema
expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
})
}
)
datasourceDescribe( it("discovers tables from any schema in search path", async () => {
{ await client.schema.createTable(`${database}.table1`, table => {
name: "POST /api/datasources/:datasourceId/schema", table.increments("id1").primary()
only: [DatabaseName.MYSQL], })
}, const res = await config.api.datasource.info(datasource)
({ config, dsProvider }) => { expect(res.tableNames).toBeDefined()
let datasource: Datasource expect(res.tableNames).toEqual(expect.arrayContaining(["table1"]))
let client: Knex })
beforeAll(async () => { it("does not mix columns from different tables", async () => {
const ds = await dsProvider() const repeated_table_name = "table_same_name"
datasource = ds.datasource! await client.schema.createTable(
client = ds.client! `${database}.${repeated_table_name}`,
}) table => {
table.increments("id").primary()
let tableName: string table.string("val1")
beforeEach(async () => { }
tableName = uniqueTableName()
})
afterEach(async () => {
await client.schema.dropTableIfExists(tableName)
})
it("recognises enum columns as options", async () => {
const enumColumnName = "status"
await client.schema.createTable(tableName, table => {
table.increments("order_id").primary()
table.string("customer_name", 100).notNullable()
table.enum(
enumColumnName,
["pending", "processing", "shipped", "delivered", "cancelled"],
{ useNative: true, enumName: `${tableName}_${enumColumnName}` }
) )
await client.schema.createTable(
`${database2}.${repeated_table_name}`,
table => {
table.increments("id2").primary()
table.string("val2")
}
)
const res = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
tablesFilter: [repeated_table_name],
})
expect(
res.datasource.entities![repeated_table_name].schema
).toBeDefined()
const schema = res.datasource.entities![repeated_table_name].schema
expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
}) })
}
)
const res = await config.api.datasource.fetchSchema({ const descriptions = datasourceDescribe({ only: [DatabaseName.MYSQL] })
datasourceId: datasource._id!,
})
const table = res.datasource.entities![tableName] if (descriptions.length) {
describe.each(descriptions)(
"POST /api/datasources/:datasourceId/schema ($dbName)",
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
expect(table).toBeDefined() beforeAll(async () => {
expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS) const ds = await dsProvider()
}) datasource = ds.datasource!
client = ds.client!
})
let tableName: string
beforeEach(async () => {
tableName = uniqueTableName()
})
afterEach(async () => {
await client.schema.dropTableIfExists(tableName)
})
it("recognises enum columns as options", async () => {
const enumColumnName = "status"
await client.schema.createTable(tableName, table => {
table.increments("order_id").primary()
table.string("customer_name", 100).notNullable()
table.enum(
enumColumnName,
["pending", "processing", "shipped", "delivered", "cancelled"],
{ useNative: true, enumName: `${tableName}_${enumColumnName}` }
)
})
const res = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const table = res.datasource.entities![tableName]
expect(table).toBeDefined()
expect(table.schema[enumColumnName].type).toEqual(FieldType.OPTIONS)
})
}
)
} }
) }

477
packages/server/src/integration-test/postgres.spec.ts
View File

@ -8,283 +8,292 @@ import {
} from "../integrations/tests/utils" } from "../integrations/tests/utils"
import { Knex } from "knex" import { Knex } from "knex"
datasourceDescribe( const mainDescriptions = datasourceDescribe({ only: [DatabaseName.POSTGRES] })
{ name: "postgres integrations", only: [DatabaseName.POSTGRES] },
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
beforeAll(async () => { if (mainDescriptions.length) {
const ds = await dsProvider() describe.each(mainDescriptions)(
datasource = ds.datasource! "/postgres integrations",
client = ds.client! ({ config, dsProvider }) => {
}) let datasource: Datasource
let client: Knex
afterAll(config.end) beforeAll(async () => {
const ds = await dsProvider()
describe("POST /api/datasources/:datasourceId/schema", () => { datasource = ds.datasource!
let tableName: string client = ds.client!
beforeEach(async () => {
tableName = generator.guid().replaceAll("-", "").substring(0, 10)
}) })
afterEach(async () => { afterAll(config.end)
await client.schema.dropTableIfExists(tableName)
})
it("recognises when a table has no primary key", async () => { describe("POST /api/datasources/:datasourceId/schema", () => {
await client.schema.createTable(tableName, table => { let tableName: string
table.increments("id", { primaryKey: false })
beforeEach(async () => {
tableName = generator.guid().replaceAll("-", "").substring(0, 10)
}) })
const response = await config.api.datasource.fetchSchema({ afterEach(async () => {
datasourceId: datasource._id!, await client.schema.dropTableIfExists(tableName)
}) })
expect(response.errors).toEqual({ it("recognises when a table has no primary key", async () => {
[tableName]: "Table must have a primary key.", await client.schema.createTable(tableName, table => {
}) table.increments("id", { primaryKey: false })
}) })
it("recognises when a table is using a reserved column name", async () => { const response = await config.api.datasource.fetchSchema({
await client.schema.createTable(tableName, table => { datasourceId: datasource._id!,
table.increments("_id").primary() })
})
const response = await config.api.datasource.fetchSchema({ expect(response.errors).toEqual({
datasourceId: datasource._id!, [tableName]: "Table must have a primary key.",
})
expect(response.errors).toEqual({
[tableName]: "Table contains invalid columns.",
})
})
it("recognises enum columns as options", async () => {
const tableName = `orders_${generator
.guid()
.replaceAll("-", "")
.substring(0, 6)}`
await client.schema.createTable(tableName, table => {
table.increments("order_id").primary()
table.string("customer_name").notNullable()
table.enum("status", ["pending", "processing", "shipped"], {
useNative: true,
enumName: `${tableName}_status`,
}) })
}) })
const response = await config.api.datasource.fetchSchema({ it("recognises when a table is using a reserved column name", async () => {
datasourceId: datasource._id!, await client.schema.createTable(tableName, table => {
table.increments("_id").primary()
})
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
expect(response.errors).toEqual({
[tableName]: "Table contains invalid columns.",
})
}) })
const table = response.datasource.entities?.[tableName] it("recognises enum columns as options", async () => {
const tableName = `orders_${generator
.guid()
.replaceAll("-", "")
.substring(0, 6)}`
expect(table).toBeDefined() await client.schema.createTable(tableName, table => {
expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS) table.increments("order_id").primary()
}) table.string("customer_name").notNullable()
}) table.enum("status", ["pending", "processing", "shipped"], {
useNative: true,
enumName: `${tableName}_status`,
})
})
describe("check custom column types", () => { const response = await config.api.datasource.fetchSchema({
beforeAll(async () => { datasourceId: datasource._id!,
await client.schema.createTable("binaryTable", table => { })
table.binary("id").primary()
table.string("column1") const table = response.datasource.entities?.[tableName]
table.integer("column2")
expect(table).toBeDefined()
expect(table?.schema["status"].type).toEqual(FieldType.OPTIONS)
}) })
}) })
it("should handle binary columns", async () => { describe("check custom column types", () => {
const response = await config.api.datasource.fetchSchema({ beforeAll(async () => {
datasourceId: datasource._id!, await client.schema.createTable("binaryTable", table => {
table.binary("id").primary()
table.string("column1")
table.integer("column2")
})
}) })
expect(response.datasource.entities).toBeDefined()
const table = response.datasource.entities?.["binaryTable"]
expect(table).toBeDefined()
expect(table?.schema.id.externalType).toBe("bytea")
const row = await config.api.row.save(table?._id!, {
id: "1111",
column1: "hello",
column2: 222,
})
expect(row._id).toBeDefined()
const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
expect(JSON.parse(decoded)[0]).toBe("1111")
})
})
describe("check fetching null/not null table", () => { it("should handle binary columns", async () => {
beforeAll(async () => { const response = await config.api.datasource.fetchSchema({
await client.schema.createTable("nullableTable", table => { datasourceId: datasource._id!,
table.increments("order_id").primary() })
table.integer("order_number").notNullable() expect(response.datasource.entities).toBeDefined()
const table = response.datasource.entities?.["binaryTable"]
expect(table).toBeDefined()
expect(table?.schema.id.externalType).toBe("bytea")
const row = await config.api.row.save(table?._id!, {
id: "1111",
column1: "hello",
column2: 222,
})
expect(row._id).toBeDefined()
const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
expect(JSON.parse(decoded)[0]).toBe("1111")
}) })
}) })
it("should be able to change the table to allow nullable and refetch this", async () => { describe("check fetching null/not null table", () => {
const response = await config.api.datasource.fetchSchema({ beforeAll(async () => {
datasourceId: datasource._id!, await client.schema.createTable("nullableTable", table => {
}) table.increments("order_id").primary()
const entities = response.datasource.entities table.integer("order_number").notNullable()
expect(entities).toBeDefined() })
const nullableTable = entities?.["nullableTable"]
expect(nullableTable).toBeDefined()
expect(
nullableTable?.schema["order_number"].constraints?.presence
).toEqual(true)
// need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase
// is aware of - therefore we can try to fetch and make sure BB updates correctly
await client.schema.alterTable("nullableTable", table => {
table.setNullable("order_number")
}) })
const responseAfter = await config.api.datasource.fetchSchema({ it("should be able to change the table to allow nullable and refetch this", async () => {
datasourceId: datasource._id!, const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entities = response.datasource.entities
expect(entities).toBeDefined()
const nullableTable = entities?.["nullableTable"]
expect(nullableTable).toBeDefined()
expect(
nullableTable?.schema["order_number"].constraints?.presence
).toEqual(true)
// need to perform these calls raw to the DB so that the external state of the DB differs to what Budibase
// is aware of - therefore we can try to fetch and make sure BB updates correctly
await client.schema.alterTable("nullableTable", table => {
table.setNullable("order_number")
})
const responseAfter = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
})
const entitiesAfter = responseAfter.datasource.entities
expect(entitiesAfter).toBeDefined()
const nullableTableAfter = entitiesAfter?.["nullableTable"]
expect(nullableTableAfter).toBeDefined()
expect(
nullableTableAfter?.schema["order_number"].constraints?.presence
).toBeUndefined()
}) })
const entitiesAfter = responseAfter.datasource.entities
expect(entitiesAfter).toBeDefined()
const nullableTableAfter = entitiesAfter?.["nullableTable"]
expect(nullableTableAfter).toBeDefined()
expect(
nullableTableAfter?.schema["order_number"].constraints?.presence
).toBeUndefined()
}) })
})
describe("money field 💰", () => { describe("money field 💰", () => {
const tableName = "moneytable" const tableName = "moneytable"
let table: Table let table: Table
beforeAll(async () => { beforeAll(async () => {
await client.raw(` await client.raw(`
CREATE TABLE ${tableName} ( CREATE TABLE ${tableName} (
id serial PRIMARY KEY, id serial PRIMARY KEY,
price money price money
) )
`) `)
const response = await config.api.datasource.fetchSchema({ const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!, datasourceId: datasource._id!,
}) })
table = response.datasource.entities![tableName] table = response.datasource.entities![tableName]
})
it("should be able to import a money field", async () => {
expect(table).toBeDefined()
expect(table?.schema.price.type).toBe(FieldType.NUMBER)
})
it("should be able to search a money field", async () => {
await config.api.row.bulkImport(table._id!, {
rows: [{ price: 200 }, { price: 300 }],
}) })
const { rows } = await config.api.row.search(table._id!, { it("should be able to import a money field", async () => {
query: { expect(table).toBeDefined()
equal: { expect(table?.schema.price.type).toBe(FieldType.NUMBER)
price: 200, })
it("should be able to search a money field", async () => {
await config.api.row.bulkImport(table._id!, {
rows: [{ price: 200 }, { price: 300 }],
})
const { rows } = await config.api.row.search(table._id!, {
query: {
equal: {
price: 200,
},
}, },
}, })
expect(rows).toHaveLength(1)
expect(rows[0].price).toBe("200.00")
})
it("should be able to update a money field", async () => {
let row = await config.api.row.save(table._id!, { price: 200 })
expect(row.price).toBe("200.00")
row = await config.api.row.save(table._id!, { ...row, price: 300 })
expect(row.price).toBe("300.00")
row = await config.api.row.save(table._id!, {
...row,
price: "400.00",
})
expect(row.price).toBe("400.00")
}) })
expect(rows).toHaveLength(1)
expect(rows[0].price).toBe("200.00")
}) })
}
)
it("should be able to update a money field", async () => { const descriptions = datasourceDescribe({ only: [DatabaseName.POSTGRES] })
let row = await config.api.row.save(table._id!, { price: 200 })
expect(row.price).toBe("200.00")
row = await config.api.row.save(table._id!, { ...row, price: 300 }) if (descriptions.length) {
expect(row.price).toBe("300.00") describe.each(descriptions)(
"Integration compatibility with postgres search_path",
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
let schema1: string
let schema2: string
row = await config.api.row.save(table._id!, { ...row, price: "400.00" }) beforeEach(async () => {
expect(row.price).toBe("400.00") const ds = await dsProvider()
}) datasource = ds.datasource!
}) const rawDatasource = ds.rawDatasource!
schema1 = generator.guid().replaceAll("-", "")
schema2 = generator.guid().replaceAll("-", "")
client = await knexClient(rawDatasource)
await client.schema.createSchema(schema1)
await client.schema.createSchema(schema2)
rawDatasource.config!.schema = `${schema1}, ${schema2}`
client = await knexClient(rawDatasource)
datasource = await config.api.datasource.create(rawDatasource)
})
afterEach(async () => {
await client.schema.dropSchema(schema1, true)
await client.schema.dropSchema(schema2, true)
})
it("discovers tables from any schema in search path", async () => {
await client.schema.createTable(`${schema1}.table1`, table => {
table.increments("id1").primary()
})
await client.schema.createTable(`${schema2}.table2`, table => {
table.increments("id2").primary()
})
const response = await config.api.datasource.info(datasource)
expect(response.tableNames).toBeDefined()
expect(response.tableNames).toEqual(
expect.arrayContaining(["table1", "table2"])
)
})
it("does not mix columns from different tables", async () => {
const repeated_table_name = "table_same_name"
await client.schema.createTable(
`${schema1}.${repeated_table_name}`,
table => {
table.increments("id").primary()
table.string("val1")
}
)
await client.schema.createTable(
`${schema2}.${repeated_table_name}`,
table => {
table.increments("id2").primary()
table.string("val2")
}
)
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
tablesFilter: [repeated_table_name],
})
expect(
response.datasource.entities?.[repeated_table_name].schema
).toBeDefined()
const schema =
response.datasource.entities?.[repeated_table_name].schema
expect(Object.keys(schema || {}).sort()).toEqual(["id", "val1"])
})
}
)
} }
) }
datasourceDescribe(
{
name: "Integration compatibility with postgres search_path",
only: [DatabaseName.POSTGRES],
},
({ config, dsProvider }) => {
let datasource: Datasource
let client: Knex
let schema1: string
let schema2: string
beforeEach(async () => {
const ds = await dsProvider()
datasource = ds.datasource!
const rawDatasource = ds.rawDatasource!
schema1 = generator.guid().replaceAll("-", "")
schema2 = generator.guid().replaceAll("-", "")
client = await knexClient(rawDatasource)
await client.schema.createSchema(schema1)
await client.schema.createSchema(schema2)
rawDatasource.config!.schema = `${schema1}, ${schema2}`
client = await knexClient(rawDatasource)
datasource = await config.api.datasource.create(rawDatasource)
})
afterEach(async () => {
await client.schema.dropSchema(schema1, true)
await client.schema.dropSchema(schema2, true)
})
it("discovers tables from any schema in search path", async () => {
await client.schema.createTable(`${schema1}.table1`, table => {
table.increments("id1").primary()
})
await client.schema.createTable(`${schema2}.table2`, table => {
table.increments("id2").primary()
})
const response = await config.api.datasource.info(datasource)
expect(response.tableNames).toBeDefined()
expect(response.tableNames).toEqual(
expect.arrayContaining(["table1", "table2"])
)
})
it("does not mix columns from different tables", async () => {
const repeated_table_name = "table_same_name"
await client.schema.createTable(
`${schema1}.${repeated_table_name}`,
table => {
table.increments("id").primary()
table.string("val1")
}
)
await client.schema.createTable(
`${schema2}.${repeated_table_name}`,
table => {
table.increments("id2").primary()
table.string("val2")
}
)
const response = await config.api.datasource.fetchSchema({
datasourceId: datasource._id!,
tablesFilter: [repeated_table_name],
})
expect(
response.datasource.entities?.[repeated_table_name].schema
).toBeDefined()
const schema = response.datasource.entities?.[repeated_table_name].schema
expect(Object.keys(schema || {}).sort()).toEqual(["id", "val1"])
})
}
)

6
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -281,8 +281,14 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
case MSSQLConfigAuthType.NTLM: { case MSSQLConfigAuthType.NTLM: {
const { domain, trustServerCertificate } = const { domain, trustServerCertificate } =
this.config.ntlmConfig || {} this.config.ntlmConfig || {}
if (!domain) {
throw Error("Domain must be provided for NTLM config")
}
clientCfg.authentication = { clientCfg.authentication = {
type: "ntlm", type: "ntlm",
// @ts-expect-error - username and password not required for NTLM
options: { options: {
domain, domain,
}, },

50
packages/server/src/integrations/snowflake.ts
View File

@ -6,7 +6,8 @@ import {
QueryType, QueryType,
SqlQuery, SqlQuery,
} from "@budibase/types" } from "@budibase/types"
import { Snowflake } from "snowflake-promise" import snowflakeSdk, { SnowflakeError } from "snowflake-sdk"
import { promisify } from "util"
interface SnowflakeConfig { interface SnowflakeConfig {
account: string account: string
@ -71,11 +72,52 @@ const SCHEMA: Integration = {
}, },
} }
class SnowflakeIntegration { class SnowflakePromise {
private client: Snowflake config: SnowflakeConfig
client?: snowflakeSdk.Connection
constructor(config: SnowflakeConfig) { constructor(config: SnowflakeConfig) {
this.client = new Snowflake(config) this.config = config
}
async connect() {
if (this.client?.isUp()) return
this.client = snowflakeSdk.createConnection(this.config)
const connectAsync = promisify(this.client.connect.bind(this.client))
return connectAsync()
}
async execute(sql: string) {
return new Promise((resolve, reject) => {
if (!this.client) {
throw Error(
"No snowflake client present to execute query. Run connect() first to initialise."
)
}
this.client.execute({
sqlText: sql,
complete: function (
err: SnowflakeError | undefined,
statementExecuted: any,
rows: any
) {
if (err) {
return reject(err)
}
resolve(rows)
},
})
})
}
}
class SnowflakeIntegration {
private client: SnowflakePromise
constructor(config: SnowflakeConfig) {
this.client = new SnowflakePromise(config)
} }
async testConnection(): Promise<ConnectionInfo> { async testConnection(): Promise<ConnectionInfo> {

57
packages/server/src/integrations/tests/utils/index.ts
View File

@ -35,7 +35,6 @@ const providers: Record<DatabaseName, DatasourceProvider> = {
} }
export interface DatasourceDescribeOpts { export interface DatasourceDescribeOpts {
name: string
only?: DatabaseName[] only?: DatabaseName[]
exclude?: DatabaseName[] exclude?: DatabaseName[]
} }
@ -102,16 +101,12 @@ function createDummyTest() {
}) })
} }
export function datasourceDescribe( export function datasourceDescribe(opts: DatasourceDescribeOpts) {
opts: DatasourceDescribeOpts,
cb: (args: DatasourceDescribeReturn) => void
) {
if (process.env.DATASOURCE === "none") { if (process.env.DATASOURCE === "none") {
createDummyTest() createDummyTest()
return
} }
const { name, only, exclude } = opts const { only, exclude } = opts
if (only && exclude) { if (only && exclude) {
throw new Error("you can only supply one of 'only' or 'exclude'") throw new Error("you can only supply one of 'only' or 'exclude'")
@ -130,36 +125,28 @@ export function datasourceDescribe(
if (databases.length === 0) { if (databases.length === 0) {
createDummyTest() createDummyTest()
return
} }
describe.each(databases)(name, name => { const config = new TestConfiguration()
const config = new TestConfiguration() return databases.map(dbName => ({
dbName,
afterAll(() => { config,
config.end() dsProvider: () => createDatasources(config, dbName),
}) isInternal: dbName === DatabaseName.SQS,
isExternal: dbName !== DatabaseName.SQS,
cb({ isSql: [
name, DatabaseName.MARIADB,
config, DatabaseName.MYSQL,
dsProvider: () => createDatasources(config, name), DatabaseName.POSTGRES,
isInternal: name === DatabaseName.SQS, DatabaseName.SQL_SERVER,
isExternal: name !== DatabaseName.SQS, DatabaseName.ORACLE,
isSql: [ ].includes(dbName),
DatabaseName.MARIADB, isMySQL: dbName === DatabaseName.MYSQL,
DatabaseName.MYSQL, isPostgres: dbName === DatabaseName.POSTGRES,
DatabaseName.POSTGRES, isMongodb: dbName === DatabaseName.MONGODB,
DatabaseName.SQL_SERVER, isMSSQL: dbName === DatabaseName.SQL_SERVER,
DatabaseName.ORACLE, isOracle: dbName === DatabaseName.ORACLE,
].includes(name), }))
isMySQL: name === DatabaseName.MYSQL,
isPostgres: name === DatabaseName.POSTGRES,
isMongodb: name === DatabaseName.MONGODB,
isMSSQL: name === DatabaseName.SQL_SERVER,
isOracle: name === DatabaseName.ORACLE,
})
})
} }
function getDatasource( function getDatasource(

4
packages/server/src/middleware/publicApi.ts
View File

@ -1,8 +1,8 @@
import { constants, utils } from "@budibase/backend-core" import { constants, utils } from "@budibase/backend-core"
import { BBContext } from "@budibase/types" import { Ctx } from "@budibase/types"
export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) { export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) {
return async (ctx: BBContext, next: any) => { return async (ctx: Ctx, next: any) => {
const appId = await utils.getAppIdFromCtx(ctx) const appId = await utils.getAppIdFromCtx(ctx)
if (requiresAppId && !appId) { if (requiresAppId && !appId) {
ctx.throw( ctx.throw(

364
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -19,202 +19,206 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures"
// These test cases are only for things that cannot be tested through the API // These test cases are only for things that cannot be tested through the API
// (e.g. limiting searches to returning specific fields). If it's possible to // (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead. // test through the API, it should be done there instead.
datasourceDescribe( const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
{ name: "search sdk (%s)", exclude: [DatabaseName.MONGODB] },
({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
let table: Table
beforeAll(async () => { if (descriptions.length) {
const ds = await dsProvider() describe.each(descriptions)(
datasource = ds.datasource "search sdk ($dbName)",
}) ({ config, dsProvider, isInternal }) => {
let datasource: Datasource | undefined
let table: Table
beforeEach(async () => { beforeAll(async () => {
const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata = const ds = await dsProvider()
isInternal datasource = ds.datasource
? {
name: "id",
type: FieldType.AUTO,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
}
: {
name: "id",
type: FieldType.NUMBER,
autocolumn: true,
}
table = await config.api.table.save(
tableForDatasource(datasource, {
primary: ["id"],
schema: {
id: idFieldSchema,
name: {
name: "name",
type: FieldType.STRING,
},
surname: {
name: "surname",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
address: {
name: "address",
type: FieldType.STRING,
},
},
})
)
for (let i = 0; i < 10; i++) {
await config.api.row.save(table._id!, {
name: generator.first(),
surname: generator.last(),
age: generator.age(),
address: generator.address(),
})
}
})
afterAll(async () => {
config.end()
})
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(rows).toHaveLength(10)
for (const row of rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
}) })
})
!isInternal && beforeEach(async () => {
it("will decode _id in oneOf query", async () => { const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata =
await config.doInContext(config.appId, async () => { isInternal
const result = await search({ ? {
tableId: table._id!, name: "id",
query: { type: FieldType.AUTO,
oneOf: { subtype: AutoFieldSubType.AUTO_ID,
_id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], autocolumn: true,
}
: {
name: "id",
type: FieldType.NUMBER,
autocolumn: true,
}
table = await config.api.table.save(
tableForDatasource(datasource, {
primary: ["id"],
schema: {
id: idFieldSchema,
name: {
name: "name",
type: FieldType.STRING,
},
surname: {
name: "surname",
type: FieldType.STRING,
},
age: {
name: "age",
type: FieldType.NUMBER,
},
address: {
name: "address",
type: FieldType.STRING,
}, },
}, },
}) })
)
expect(result.rows).toHaveLength(3) for (let i = 0; i < 10; i++) {
expect(result.rows.map(row => row.id)).toEqual( await config.api.row.save(table._id!, {
expect.arrayContaining([1, 4, 8]) name: generator.first(),
) surname: generator.last(),
}) age: generator.age(),
}) address: generator.address(),
})
it("does not allow accessing hidden fields", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("surname")
expect(keys).toContain("address")
expect(keys).not.toContain("age")
} }
}) })
})
it("does not allow accessing hidden fields even if requested", async () => { afterAll(async () => {
await config.doInContext(config.appId, async () => { config.end()
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).not.toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
}) })
})
it.each([ it("querying by fields will always return data attribute columns", async () => {
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => { await config.doInContext(config.appId, async () => {
const { rows } = await search({ const { rows } = await search({
tableId: table._id!, tableId: table._id!,
query: { query: {},
$or: { fields: ["name", "age"],
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
},
{ equal: { id: 7 } },
],
},
},
fields: queryFields,
}) })
expect(rows).toHaveLength(expectedRows) expect(rows).toHaveLength(10)
for (const row of rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
}) })
} })
)
} !isInternal &&
) it("will decode _id in oneOf query", async () => {
await config.doInContext(config.appId, async () => {
const result = await search({
tableId: table._id!,
query: {
oneOf: {
_id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
},
},
})
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual(
expect.arrayContaining([1, 4, 8])
)
})
})
it("does not allow accessing hidden fields", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).toContain("surname")
expect(keys).toContain("address")
expect(keys).not.toContain("age")
}
})
})
it("does not allow accessing hidden fields even if requested", async () => {
await config.doInContext(config.appId, async () => {
await config.api.table.save({
...table,
schema: {
...table.schema,
name: {
...table.schema.name,
visible: true,
},
age: {
...table.schema.age,
visible: false,
},
},
})
const result = await search({
tableId: table._id!,
query: {},
fields: ["name", "age"],
})
expect(result.rows).toHaveLength(10)
for (const row of result.rows) {
const keys = Object.keys(row)
expect(keys).toContain("name")
expect(keys).not.toContain("age")
expect(keys).not.toContain("surname")
expect(keys).not.toContain("address")
}
})
})
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
},
{ equal: { id: 7 } },
],
},
},
fields: queryFields,
})
expect(rows).toHaveLength(expectedRows)
})
}
)
}
)
}

5
packages/server/src/sdk/app/tables/getters.ts
View File

@ -78,8 +78,11 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
} }
async function getAllExternalTables(): Promise<Table[]> { async function getAllExternalTables(): Promise<Table[]> {
// this is all datasources, we'll need to filter out internal
const datasources = await sdk.datasources.fetch({ enriched: true }) const datasources = await sdk.datasources.fetch({ enriched: true })
const allEntities = datasources.map(datasource => datasource.entities) const allEntities = datasources
.filter(datasource => datasource._id !== INTERNAL_TABLE_SOURCE_ID)
.map(datasource => datasource.entities)
let final: Table[] = [] let final: Table[] = []
for (let entities of allEntities) { for (let entities of allEntities) {
if (entities) { if (entities) {

29
packages/server/src/sdk/app/views/index.ts
View File

@ -26,6 +26,7 @@ import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./internal" import * as internal from "./internal"
import * as external from "./external" import * as external from "./external"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import { ensureQueryUISet } from "./utils"
function pickApi(tableId: any) { function pickApi(tableId: any) {
if (isExternalTableID(tableId)) { if (isExternalTableID(tableId)) {
@ -44,6 +45,24 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
return pickApi(tableId).getEnriched(viewId) return pickApi(tableId).getEnriched(viewId)
} }
export async function getAllEnriched(): Promise<ViewV2Enriched[]> {
const tables = await sdk.tables.getAllTables()
let views: ViewV2Enriched[] = []
for (let table of tables) {
if (!table.views || Object.keys(table.views).length === 0) {
continue
}
const v2Views = Object.values(table.views).filter(isV2)
const enrichedViews = await Promise.all(
v2Views.map(view =>
enrichSchema(ensureQueryUISet(view), table.schema, tables)
)
)
views = views.concat(enrichedViews)
}
return views
}
export async function getTable(view: string | ViewV2): Promise<Table> { export async function getTable(view: string | ViewV2): Promise<Table> {
const viewId = typeof view === "string" ? view : view.id const viewId = typeof view === "string" ? view : view.id
const cached = context.getTableForView(viewId) const cached = context.getTableForView(viewId)
@ -333,13 +352,19 @@ export function allowedFields(
export async function enrichSchema( export async function enrichSchema(
view: ViewV2, view: ViewV2,
tableSchema: TableSchema tableSchema: TableSchema,
tables?: Table[]
): Promise<ViewV2Enriched> { ): Promise<ViewV2Enriched> {
async function populateRelTableSchema( async function populateRelTableSchema(
tableId: string, tableId: string,
viewFields: Record<string, RelationSchemaField> viewFields: Record<string, RelationSchemaField>
) { ) {
const relTable = await sdk.tables.getTable(tableId) let relTable = tables
? tables?.find(t => t._id === tableId)
: await sdk.tables.getTable(tableId)
if (!relTable) {
throw new Error("Cannot enrich relationship, table not found")
}
const result: Record<string, ViewV2ColumnEnriched> = {} const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) { for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName] const relTableField = relTable.schema[relTableFieldName]

5
packages/server/src/startup/index.ts
View File

@ -28,6 +28,7 @@ import Koa from "koa"
import { Server } from "http" import { Server } from "http"
import { AddressInfo } from "net" import { AddressInfo } from "net"
import fs from "fs" import fs from "fs"
import bson from "bson"
let STARTUP_RAN = false let STARTUP_RAN = false
@ -193,6 +194,10 @@ export async function startup(
}) })
} }
if (coreEnv.BSON_BUFFER_SIZE) {
bson.setInternalBufferSize(coreEnv.BSON_BUFFER_SIZE)
}
console.log("Initialising JS runner") console.log("Initialising JS runner")
jsRunner.init() jsRunner.init()
} }

7
packages/server/src/tests/utilities/api/viewV2.ts
View File

@ -5,6 +5,7 @@ import {
SearchViewRowRequest, SearchViewRowRequest,
PaginatedSearchRowResponse, PaginatedSearchRowResponse,
ViewResponseEnriched, ViewResponseEnriched,
ViewFetchResponseEnriched,
} from "@budibase/types" } from "@budibase/types"
import { Expectations, TestAPI } from "./base" import { Expectations, TestAPI } from "./base"
@ -49,6 +50,12 @@ export class ViewV2API extends TestAPI {
.data .data
} }
fetch = async (expectations?: Expectations) => {
return await this._get<ViewFetchResponseEnriched>(`/api/v2/views`, {
expectations,
})
}
search = async ( search = async (
viewId: string, viewId: string,
params?: SearchViewRowRequest, params?: SearchViewRowRequest,

16
packages/server/src/threads/automation.ts
View File

@ -385,7 +385,7 @@ class Orchestrator {
stepIdx: number, stepIdx: number,
pathIdx?: number pathIdx?: number
): Promise<number> { ): Promise<number> {
await processObject(loopStep.inputs, this.processContext(this.context)) await processObject(loopStep.inputs, this.mergeContexts(this.context))
const iterations = getLoopIterations(loopStep) const iterations = getLoopIterations(loopStep)
let stepToLoopIndex = stepIdx + 1 let stepToLoopIndex = stepIdx + 1
let pathStepIdx = (pathIdx || stepIdx) + 1 let pathStepIdx = (pathIdx || stepIdx) + 1
@ -573,14 +573,14 @@ class Orchestrator {
for (const [field, value] of Object.entries(filters[filterKey])) { for (const [field, value] of Object.entries(filters[filterKey])) {
const fromContext = processStringSync( const fromContext = processStringSync(
field, field,
this.processContext(this.context) this.mergeContexts(this.context)
) )
toFilter[field] = fromContext toFilter[field] = fromContext
if (typeof value === "string" && findHBSBlocks(value).length > 0) { if (typeof value === "string" && findHBSBlocks(value).length > 0) {
const processedVal = processStringSync( const processedVal = processStringSync(
value, value,
this.processContext(this.context) this.mergeContexts(this.context)
) )
filters[filterKey][field] = processedVal filters[filterKey][field] = processedVal
@ -637,7 +637,7 @@ class Orchestrator {
const stepFn = await this.getStepFunctionality(step.stepId) const stepFn = await this.getStepFunctionality(step.stepId)
let inputs = await processObject( let inputs = await processObject(
originalStepInput, originalStepInput,
this.processContext(this.context) this.mergeContexts(this.context)
) )
inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
@ -645,7 +645,7 @@ class Orchestrator {
inputs: inputs, inputs: inputs,
appId: this.appId, appId: this.appId,
emitter: this.emitter, emitter: this.emitter,
context: this.context, context: this.mergeContexts(this.context),
}) })
this.handleStepOutput(step, outputs, loopIteration) this.handleStepOutput(step, outputs, loopIteration)
} }
@ -665,8 +665,8 @@ class Orchestrator {
return null return null
} }
private processContext(context: AutomationContext) { private mergeContexts(context: AutomationContext) {
const processContext = { const mergeContexts = {
...context, ...context,
steps: { steps: {
...context.steps, ...context.steps,
@ -674,7 +674,7 @@ class Orchestrator {
...context.stepsByName, ...context.stepsByName,
}, },
} }
return processContext return mergeContexts
} }
private handleStepOutput( private handleStepOutput(

22
packages/server/src/threads/query.ts
View File

@ -136,21 +136,23 @@ class QueryRunner {
pagination = output.pagination pagination = output.pagination
} }
// transform as required // We avoid invoking the transformer if it's trivial because there is a cost
if (transformer) { // to passing data in and out of the isolate, especially for MongoDB where
// we have to bson serialise/deserialise the data.
const hasTransformer =
transformer != null &&
transformer.length > 0 &&
transformer.trim() !== "return data" &&
transformer.trim() !== "return data;"
if (transformer && hasTransformer) {
transformer = iifeWrapper(transformer) transformer = iifeWrapper(transformer)
let vm = new IsolatedVM() let vm = new IsolatedVM()
if (datasource.source === SourceName.MONGODB) { if (datasource.source === SourceName.MONGODB) {
vm = vm.withParsingBson(rows) vm = vm.withParsingBson(rows)
} }
const ctx = { data: rows, params: enrichedParameters }
const ctx = { rows = vm.withContext(ctx, () => vm.execute(transformer!))
data: rows,
params: enrichedParameters,
}
if (transformer != null) {
rows = vm.withContext(ctx, () => vm.execute(transformer!))
}
} }
// if the request fails we retry once, invalidating the cached value // if the request fails we retry once, invalidating the cached value

6
packages/server/src/utilities/fileSystem/processor.ts
View File

@ -1,4 +1,4 @@
import jimp from "jimp" import { Jimp } from "jimp"
const FORMATS = { const FORMATS = {
IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"], IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
@ -6,8 +6,8 @@ const FORMATS = {
function processImage(file: { path: string }) { function processImage(file: { path: string }) {
// this will overwrite the temp file // this will overwrite the temp file
return jimp.read(file.path).then(img => { return Jimp.read(file.path).then(img => {
return img.resize(300, jimp.AUTO).write(file.path) return img.resize({ w: 256 }).write(file.path as `${string}.${string}`)
}) })
} }

1
packages/server/tsconfig.json
View File

@ -1,6 +1,7 @@
{ {
"extends": "./tsconfig.build.json", "extends": "./tsconfig.build.json",
"compilerOptions": { "compilerOptions": {
"lib": ["es2020", "dom"],
"composite": true, "composite": true,
"baseUrl": "." "baseUrl": "."
}, },

4
packages/string-templates/.npmignore Normal file
View File

@ -0,0 +1,4 @@
*
!dist/**/*
dist/tsconfig.build.tsbuildinfo
!package.json

Some files were not shown because too many files have changed in this diff Show More