MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into new-datepicker

This commit is contained in:
Andrew Kingston 2024-04-23 12:06:06 +01:00
parent bb575f1ccd b080821f29
commit 0aeddfa029
150 changed files with 2319 additions and 1359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.eslintrc.json
View File

@ -42,7 +42,17 @@
},
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
"local-rules/no-budibase-imports": "error",
"local-rules/no-console-error": "error",
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"local-rules/no-budibase-imports": "error"
}
},
@ -59,7 +69,15 @@
},
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"local-rules/no-test-com": "error",
"local-rules/email-domain-example-com": "error",
"no-console": "warn",
@ -89,7 +107,8 @@
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_"
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"import/no-relative-packages": "error",

3
.vscode/settings.json vendored
View File

@ -24,5 +24,8 @@
},
"[svelte]": {
"editor.defaultFormatter": "svelte.svelte-vscode"
},
"[handlebars]": {
"editor.formatOnSave": false
}
}

2
charts/budibase/templates/app-service-deployment.yaml
View File

@ -106,6 +106,8 @@ spec:
value: {{ .Values.services.objectStore.globalBucketName | quote }}
- name: BACKUPS_BUCKET_NAME
value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: TEMP_BUCKET_NAME
value: {{ .Values.globals.tempBucketName | quote }}
- name: PORT
value: {{ .Values.services.apps.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}

2
charts/budibase/templates/automation-worker-service-deployment.yaml
View File

@ -107,6 +107,8 @@ spec:
value: {{ .Values.services.objectStore.globalBucketName | quote }}
- name: BACKUPS_BUCKET_NAME
value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: TEMP_BUCKET_NAME
value: {{ .Values.globals.tempBucketName | quote }}
- name: PORT
value: {{ .Values.services.automationWorkers.port | quote }}
{{ if .Values.services.worker.publicApiRateLimitPerSecond }}

2
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -106,6 +106,8 @@ spec:
value: {{ .Values.services.objectStore.globalBucketName | quote }}
- name: BACKUPS_BUCKET_NAME
value: {{ .Values.services.objectStore.backupsBucketName | quote }}
- name: TEMP_BUCKET_NAME
value: {{ .Values.globals.tempBucketName | quote }}
- name: PORT
value: {{ .Values.services.worker.port | quote }}
- name: MULTI_TENANCY

3
charts/budibase/values.yaml
View File

@ -121,6 +121,9 @@ globals:
# to the old value for the duration of the rotation.
jwtSecretFallback: ""
## -- If using S3 the bucket name to be used for storing temporary files
tempBucketName: ""
smtp:
# -- Whether to enable SMTP or not.
enabled: false

21
eslint-local-rules/index.js
View File

@ -1,4 +1,25 @@
module.exports = {
"no-console-error": {
create: function(context) {
return {
CallExpression(node) {
if (
node.callee.type === "MemberExpression" &&
node.callee.object.name === "console" &&
node.callee.property.name === "error" &&
node.arguments.length === 1 &&
node.arguments[0].name &&
node.arguments[0].name.startsWith("err")
) {
context.report({
node,
message: 'Using console.error(err) on its own is not allowed. Either provide context to the error (console.error(msg, err)) or throw it.',
})
}
},
};
},
},
"no-budibase-imports": {
create: function (context) {
return {

3
hosting/.env
View File

@ -17,6 +17,7 @@ APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
COUCH_DB_SQS_PORT=4006
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
PLUGINS_DIR=
ROLLING_LOG_MAX_SIZE=
ROLLING_LOG_MAX_SIZE=

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.23.5",
"version": "2.23.11",
"npmClient": "yarn",
"packages": [
"packages/*",

5
nx.json
View File

@ -9,10 +9,7 @@
},
"targetDefaults": {
"build": {
"inputs": [
"{workspaceRoot}/scripts/build.js",
"{workspaceRoot}/lerna.json"
]
"inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"]
}
}
}

2
package.json
View File

@ -59,7 +59,7 @@
"dev:camunda": "./scripts/deploy-camunda.sh",
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
"dev:docker": "./scripts/devDocker.sh",
"test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
"lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",

2
packages/account-portal

@ -1 +1 @@
Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344
Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964

2
packages/backend-core/src/auth/auth.ts
View File

@ -64,7 +64,6 @@ async function refreshOIDCAccessToken(
}
strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
} catch (err) {
console.error(err)
throw new Error("Could not refresh OAuth Token")
}
@ -99,7 +98,6 @@ async function refreshGoogleAccessToken(
ssoSaveUserNoOp
)
} catch (err: any) {
console.error(err)
throw new Error(
`Error constructing OIDC refresh strategy: message=${err.message}`
)

14
packages/backend-core/src/db/lucene.ts
View File

@ -8,19 +8,9 @@ import {
SearchParams,
WithRequired,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
const QUERY_START_REGEX = /\d[0-9]*:/g
export function removeKeyNumbering(key: any): string {
if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
const parts = key.split(":")
// remove the number
parts.shift()
return parts.join(":")
} else {
return key
}
}
export const removeKeyNumbering = dataFilters.removeKeyNumbering
/**
* Class to build lucene query URLs.

4
packages/backend-core/src/environment.ts
View File

@ -29,6 +29,7 @@ const DefaultBucketName = {
TEMPLATES: "templates",
GLOBAL: "global",
PLUGINS: "plugins",
TEMP: "tmp-file-attachments",
}
const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
@ -107,7 +108,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
@ -146,6 +147,7 @@ const environment = {
process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
PLUGIN_BUCKET_NAME:
process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
USE_COUCH: process.env.USE_COUCH || true,
MOCK_REDIS: process.env.MOCK_REDIS,
DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,

2
packages/backend-core/src/middleware/authenticated.ts
View File

@ -138,7 +138,6 @@ export default function (
} catch (err: any) {
authenticated = false
console.error(`Auth Error: ${err.message}`)
console.error(err)
// remove the cookie as the user does not exist anymore
clearCookie(ctx, Cookie.Auth)
}
@ -187,7 +186,6 @@ export default function (
}
} catch (err: any) {
console.error(`Auth Error: ${err.message}`)
console.error(err)
// invalid token, clear the cookie
if (err?.name === "JsonWebTokenError") {
clearCookie(ctx, Cookie.Auth)

2
packages/backend-core/src/middleware/errorHandling.ts
View File

@ -12,7 +12,7 @@ export async function errorHandling(ctx: any, next: any) {
if (status >= 400 && status < 500) {
console.warn(err)
} else {
console.error(err)
console.error("Got 400 response code", err)
}
let error: APIError = {

1
packages/backend-core/src/middleware/passport/sso/google.ts
View File

@ -68,7 +68,6 @@ export async function strategyFactory(
verify
)
} catch (err: any) {
console.error(err)
throw new Error(`Error constructing google authentication strategy: ${err}`)
}
}

2
packages/backend-core/src/middleware/passport/sso/oidc.ts
View File

@ -103,7 +103,6 @@ export async function strategyFactory(
strategy.name = "oidc"
return strategy
} catch (err: any) {
console.error(err)
throw new Error(`Error constructing OIDC authentication strategy - ${err}`)
}
}
@ -142,7 +141,6 @@ export async function fetchStrategyConfig(
callbackURL: callbackUrl,
}
} catch (err) {
console.error(err)
throw new Error(
`Error constructing OIDC authentication configuration - ${err}`
)

1
packages/backend-core/src/migrations/migrations.ts
View File

@ -26,7 +26,6 @@ export const getMigrationsDoc = async (db: any) => {
if (err.status && err.status === 404) {
return { _id: DocumentType.MIGRATIONS }
} else {
console.error(err)
throw err
}
}

121
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -7,31 +7,41 @@ import tar from "tar-fs"
import zlib from "zlib"
import { promisify } from "util"
import { join } from "path"
import fs, { ReadStream } from "fs"
import fs, { PathLike, ReadStream } from "fs"
import env from "../environment"
import { budibaseTempDir } from "./utils"
import { bucketTTLConfig, budibaseTempDir } from "./utils"
import { v4 } from "uuid"
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
import fsp from "fs/promises"
const streamPipeline = promisify(stream.pipeline)
// use this as a temporary store of buckets that are being created
const STATE = {
bucketCreationPromises: {},
}
const signedFilePrefix = "/files/signed"
type ListParams = {
ContinuationToken?: string
}
type UploadParams = {
type BaseUploadParams = {
bucket: string
filename: string
path: string
type?: string | null
// can be undefined, we will remove it
metadata?: {
[key: string]: string | undefined
}
metadata?: { [key: string]: string | undefined }
body?: ReadableStream | Buffer
ttl?: number
addTTL?: boolean
extra?: any
}
type UploadParams = BaseUploadParams & {
path?: string | PathLike
}
type StreamUploadParams = BaseUploadParams & {
stream: ReadStream
}
const CONTENT_TYPE_MAP: any = {
@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = {
js: "application/javascript",
json: "application/json",
gz: "application/gzip",
svg: "image/svg+xml",
form: "multipart/form-data",
}
const STRING_CONTENT_TYPES = [
@ -105,7 +117,10 @@ export function ObjectStore(
* Given an object store and a bucket name this will make sure the bucket exists,
* if it does not exist then it will create it.
*/
export async function makeSureBucketExists(client: any, bucketName: string) {
export async function createBucketIfNotExists(
client: any,
bucketName: string
): Promise<{ created: boolean; exists: boolean }> {
bucketName = sanitizeBucket(bucketName)
try {
await client
@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
Bucket: bucketName,
})
.promise()
return { created: false, exists: true }
} catch (err: any) {
const promises: any = STATE.bucketCreationPromises
const doesntExist = err.statusCode === 404,
noAccess = err.statusCode === 403
if (promises[bucketName]) {
await promises[bucketName]
return { created: false, exists: true }
} else if (doesntExist || noAccess) {
if (doesntExist) {
// bucket doesn't exist create it
promises[bucketName] = client
.createBucket({
Bucket: bucketName,
@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
.promise()
await promises[bucketName]
delete promises[bucketName]
return { created: true, exists: false }
} else {
throw new Error("Access denied to object store bucket." + err)
}
} else {
throw new Error("Unable to write to object store bucket.")
}
}
}
/**
* Uploads the contents of a file given the required parameters, useful when
* temp files in use (for example file uploaded as an attachment).
@ -146,12 +164,22 @@ export async function upload({
path,
type,
metadata,
body,
ttl,
}: UploadParams) {
const extension = filename.split(".").pop()
const fileBytes = fs.readFileSync(path)
const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
let ttlConfig = bucketTTLConfig(bucketName, ttl)
if (objectStore.putBucketLifecycleConfiguration) {
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
}
}
let contentType = type
if (!contentType) {
@ -174,6 +202,7 @@ export async function upload({
}
config.Metadata = metadata
}
return objectStore.upload(config).promise()
}
@ -181,14 +210,24 @@ export async function upload({
* Similar to the upload function but can be used to send a file stream
* through to the object store.
*/
export async function streamUpload(
bucketName: string,
filename: string,
stream: ReadStream | ReadableStream,
extra = {}
) {
export async function streamUpload({
bucket: bucketName,
stream,
filename,
type,
extra,
ttl,
}: StreamUploadParams) {
const extension = filename.split(".").pop()
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
let ttlConfig = bucketTTLConfig(bucketName, ttl)
if (objectStore.putBucketLifecycleConfiguration) {
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
}
}
// Set content type for certain known extensions
if (filename?.endsWith(".js")) {
@ -203,10 +242,18 @@ export async function streamUpload(
}
}
let contentType = type
if (!contentType) {
contentType = extension
? CONTENT_TYPE_MAP[extension.toLowerCase()]
: CONTENT_TYPE_MAP.txt
}
const params = {
Bucket: sanitizeBucket(bucketName),
Key: sanitizeKey(filename),
Body: stream,
ContentType: contentType,
...extra,
}
return objectStore.upload(params).promise()
@ -286,7 +333,7 @@ export function getPresignedUrl(
const signedUrl = new URL(url)
const path = signedUrl.pathname
const query = signedUrl.search
return `/files/signed${path}${query}`
return `${signedFilePrefix}${path}${query}`
}
}
@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
*/
export async function deleteFile(bucketName: string, filepath: string) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
await createBucketIfNotExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Key: sanitizeKey(filepath),
@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
export async function deleteFiles(bucketName: string, filepaths: string[]) {
const objectStore = ObjectStore(bucketName)
await makeSureBucketExists(objectStore, bucketName)
await createBucketIfNotExists(objectStore, bucketName)
const params = {
Bucket: bucketName,
Delete: {
@ -412,7 +459,13 @@ export async function uploadDirectory(
if (file.isDirectory()) {
uploads.push(uploadDirectory(bucketName, local, path))
} else {
uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
uploads.push(
streamUpload({
bucket: bucketName,
filename: path,
stream: fs.createReadStream(local),
})
)
}
}
await Promise.all(uploads)
@ -467,3 +520,23 @@ export async function getReadStream(
}
return client.getObject(params).createReadStream()
}
/*
Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
the bucket and the path from it
*/
export function extractBucketAndPath(
url: string
): { bucket: string; path: string } | null {
const baseUrl = url.split("?")[0]
const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
const match = baseUrl.match(regex)
if (match && match.groups) {
const { bucket, path } = match.groups
return { bucket, path }
}
return null
}

26
packages/backend-core/src/objectStore/utils.ts
View File

@ -2,6 +2,7 @@ import { join } from "path"
import { tmpdir } from "os"
import fs from "fs"
import env from "../environment"
import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
/****************************************************
* NOTE: When adding a new bucket - name *
@ -15,6 +16,7 @@ export const ObjectStoreBuckets = {
TEMPLATES: env.TEMPLATES_BUCKET_NAME,
GLOBAL: env.GLOBAL_BUCKET_NAME,
PLUGINS: env.PLUGIN_BUCKET_NAME,
TEMP: env.TEMP_BUCKET_NAME,
}
const bbTmp = join(tmpdir(), ".budibase")
@ -29,3 +31,27 @@ try {
export function budibaseTempDir() {
return bbTmp
}
export const bucketTTLConfig = (
bucketName: string,
days: number
): PutBucketLifecycleConfigurationRequest => {
const lifecycleRule = {
ID: `${bucketName}-ExpireAfter${days}days`,
Prefix: "",
Status: "Enabled",
Expiration: {
Days: days,
},
}
const lifecycleConfiguration = {
Rules: [lifecycleRule],
}
const params = {
Bucket: bucketName,
LifecycleConfiguration: lifecycleConfiguration,
}
return params
}

4
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -115,7 +115,6 @@ class InMemoryQueue implements Partial<Queue> {
* a JSON message as this is required by Bull.
* @param repeat serves no purpose for the import queue.
*/
// eslint-disable-next-line no-unused-vars
async add(data: any, opts?: JobOptions) {
const jobId = opts?.jobId?.toString()
if (jobId && this._queuedJobIds.has(jobId)) {
@ -166,8 +165,7 @@ class InMemoryQueue implements Partial<Queue> {
return []
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async removeJobs(pattern: string) {
async removeJobs(_pattern: string) {
// no-op
}

4
packages/backend-core/src/users/db.ts
View File

@ -50,6 +50,8 @@ type CreateAdminUserOpts = {
hashPassword?: boolean
requirePassword?: boolean
skipPasswordValidation?: boolean
firstName?: string
lastName?: string
}
type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
@ -517,6 +519,8 @@ export class UserDB {
global: true,
},
tenantId,
firstName: opts?.firstName,
lastName: opts?.lastName,
}
if (opts?.ssoId) {
user.ssoId = opts.ssoId

12
packages/backend-core/src/users/users.ts
View File

@ -17,8 +17,8 @@ import {
ContextUser,
CouchFindOptions,
DatabaseQueryOpts,
SearchQuery,
SearchQueryOperators,
SearchFilters,
SearchFilterOperator,
SearchUsersRequest,
User,
} from "@budibase/types"
@ -44,11 +44,11 @@ function removeUserPassword(users: User | User[]) {
return users
}
export function isSupportedUserSearch(query: SearchQuery) {
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: SearchQueryOperators.STRING, key: "email" },
{ op: SearchQueryOperators.EQUAL, key: "_id" },
{ op: SearchQueryOperators.ONE_OF, key: "_id" },
{ op: SearchFilterOperator.STRING, key: "email" },
{ op: SearchFilterOperator.EQUAL, key: "_id" },
{ op: SearchFilterOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {

3
packages/backend-core/tests/core/utilities/index.ts
View File

@ -4,3 +4,6 @@ export { generator } from "./structures"
export * as testContainerUtils from "./testContainerUtils"
export * as utils from "./utils"
export * from "./jestUtils"
import * as minio from "./minio"
export const objectStoreTestProviders = { minio }

34
packages/backend-core/tests/core/utilities/minio.ts Normal file
View File

@ -0,0 +1,34 @@
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
import env from "../../../src/environment"
let container: StartedTestContainer | undefined
class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
const logs = Wait.forListeningPorts()
await logs.waitUntilReady(container, boundPorts, startTime)
}
}
export async function start(): Promise<void> {
container = await new GenericContainer("minio/minio")
.withExposedPorts(9000)
.withCommand(["server", "/data"])
.withEnvironment({
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
})
.withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
.start()
const port = container.getMappedPort(9000)
env._set("MINIO_URL", `http://0.0.0.0:${port}`)
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
}
}

78
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -14,6 +14,7 @@
notifications,
Checkbox,
DatePicker,
DrawerContent,
} from "@budibase/bbui"
import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
import { automationStore, selectedAutomation, tables } from "stores/builder"
@ -31,13 +32,14 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte"
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
import { BindingHelpers, BindingType } from "components/common/bindings/utils"
import {
bindingsToCompletions,
hbAutocomplete,
EditorModes,
} from "components/common/CodeEditor"
import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte"
import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
import { LuceneUtils, Utils } from "@budibase/frontend-core"
import {
getSchemaForDatasourcePlus,
@ -355,7 +357,8 @@
value.customType !== "queryParams" &&
value.customType !== "cron" &&
value.customType !== "triggerSchema" &&
value.customType !== "automationFields"
value.customType !== "automationFields" &&
value.type !== "attachment"
)
}
@ -371,6 +374,15 @@
console.error(error)
}
})
const handleAttachmentParams = keyValuObj => {
let params = {}
if (keyValuObj?.length) {
for (let param of keyValuObj) {
params[param.url] = param.filename
}
}
return params
}
</script>
<div class="fields">
@ -436,21 +448,49 @@
value={inputData[key]}
options={Object.keys(table?.schema || {})}
/>
{:else if value.type === "attachment"}
<div class="attachment-field-wrapper">
<div class="label-wrapper">
<Label>{label}</Label>
</div>
<div class="attachment-field-width">
<KeyValueBuilder
on:change={e =>
onChange(
{
detail: e.detail.map(({ name, value }) => ({
url: name,
filename: value,
})),
},
key
)}
object={handleAttachmentParams(inputData[key])}
allowJS
{bindings}
keyBindings
customButtonText={"Add attachment"}
keyPlaceholder={"URL"}
valuePlaceholder={"Filename"}
/>
</div>
</div>
{:else if value.customType === "filters"}
<ActionButton on:click={drawer.show}>Define filters</ActionButton>
<Drawer bind:this={drawer} title="Filtering">
<Button cta slot="buttons" on:click={() => saveFilters(key)}>
Save
</Button>
<FilterDrawer
slot="body"
{filters}
{bindings}
{schemaFields}
datasource={{ type: "table", tableId }}
panel={AutomationBindingPanel}
on:change={e => (tempFilters = e.detail)}
/>
<DrawerContent slot="body">
<FilterBuilder
{filters}
{bindings}
{schemaFields}
datasource={{ type: "table", tableId }}
panel={AutomationBindingPanel}
on:change={e => (tempFilters = e.detail)}
/>
</DrawerContent>
</Drawer>
{:else if value.customType === "password"}
<Input
@ -649,14 +689,22 @@
}
.block-field {
display: flex; /* Use Flexbox */
display: flex;
justify-content: space-between;
flex-direction: row; /* Arrange label and field side by side */
align-items: center; /* Align vertically in the center */
gap: 10px; /* Add some space between label and field */
flex-direction: row;
align-items: center;
gap: 10px;
flex: 1;
}
.attachment-field-width {
margin-top: var(--spacing-xs);
}
.label-wrapper {
margin-top: var(--spacing-s);
}
.test :global(.drawer) {
width: 10000px !important;
}

4
packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
View File

@ -1,7 +1,7 @@
<script>
import { createEventDispatcher } from "svelte"
import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte"
import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
export let schema
export let filters
@ -40,7 +40,7 @@
onConfirm={() => dispatch("change", tempValue)}
>
<div class="wrapper">
<FilterDrawer
<FilterBuilder
allowBindings={false}
{filters}
{schemaFields}

52
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -13,6 +13,7 @@
Layout,
AbsTooltip,
} from "@budibase/bbui"
import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
@ -20,11 +21,6 @@
import {
FIELDS,
RelationshipType,
ALLOWABLE_STRING_OPTIONS,
ALLOWABLE_NUMBER_OPTIONS,
ALLOWABLE_STRING_TYPES,
ALLOWABLE_NUMBER_TYPES,
SWITCHABLE_TYPES,
PrettyRelationshipDefinitions,
DB_TYPE_EXTERNAL,
} from "constants/backend"
@ -33,21 +29,20 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
const AUTO_TYPE = FIELDS.AUTO.type
const FORMULA_TYPE = FIELDS.FORMULA.type
const LINK_TYPE = FIELDS.LINK.type
const STRING_TYPE = FIELDS.STRING.type
const NUMBER_TYPE = FIELDS.NUMBER.type
const JSON_TYPE = FIELDS.JSON.type
const DATE_TYPE = FIELDS.DATETIME.type
const USER_TYPE = FIELDS.USER.subtype
const USERS_TYPE = FIELDS.USERS.subtype
const AUTO_TYPE = FieldType.AUTO
const FORMULA_TYPE = FieldType.FORMULA
const LINK_TYPE = FieldType.LINK
const STRING_TYPE = FieldType.STRING
const NUMBER_TYPE = FieldType.NUMBER
const JSON_TYPE = FieldType.JSON
const DATE_TYPE = FieldType.DATETIME
const USER_TYPE = FieldSubtype.USER
const USERS_TYPE = FieldSubtype.USERS
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -61,8 +56,8 @@
let primaryDisplay
let indexes = [...($tables.selected.indexes || [])]
let isCreating = undefined
let relationshipPart1 = PrettyRelationshipDefinitions.Many
let relationshipPart2 = PrettyRelationshipDefinitions.One
let relationshipPart1 = PrettyRelationshipDefinitions.MANY
let relationshipPart2 = PrettyRelationshipDefinitions.ONE
let relationshipTableIdPrimary = null
let relationshipTableIdSecondary = null
let table = $tables.selected
@ -175,7 +170,7 @@
$: typeEnabled =
!originalName ||
(originalName &&
SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 &&
SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn)
const fieldDefinitions = Object.values(FIELDS).reduce(
@ -367,16 +362,15 @@
}
function getAllowedTypes() {
if (
originalName &&
ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_STRING_OPTIONS
} else if (
originalName &&
ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_NUMBER_OPTIONS
if (originalName) {
const possibleTypes = (
SWITCHABLE_TYPES[field.type] || [editableColumn.type]
).map(t => t.toLowerCase())
return Object.entries(FIELDS)
.filter(([fieldType]) =>
possibleTypes.includes(fieldType.toLowerCase())
)
.map(([_, fieldDefinition]) => fieldDefinition)
}
const isUsers =
@ -633,7 +627,7 @@
/>
</div>
</div>
{:else if editableColumn.type === FieldType.LINK}
{:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
<RelationshipSelector
bind:relationshipPart1
bind:relationshipPart2

9
packages/builder/src/components/common/Dropzone.svelte
View File

@ -27,14 +27,6 @@
return []
}
}
async function deleteAttachments(fileList) {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
</script>
<Dropzone
@ -42,6 +34,5 @@
{label}
{...$$restProps}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>

1
packages/builder/src/components/common/users/PasswordRepeatInput.svelte
View File

@ -9,7 +9,6 @@
"",
requiredValidator
)
// eslint-disable-next-line no-unused-vars
const [repeatPassword, _, repeatTouched] = createValidationStore(
"",
requiredValidator

84
packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte Normal file
View File

@ -0,0 +1,84 @@
<script>
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
import { dataFilters } from "@budibase/shared-core"
import { FilterBuilder } from "@budibase/frontend-core"
import { createEventDispatcher, onMount } from "svelte"
export let schemaFields
export let filters = []
export let bindings = []
export let panel = ClientBindingPanel
export let allowBindings = true
export let datasource
const dispatch = createEventDispatcher()
let rawFilters
$: parseFilters(rawFilters)
$: dispatch("change", enrichFilters(rawFilters))
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
rawFilters = (filters || []).map(filter => {
const { field } = filter
let newFilter = { ...filter }
delete newFilter.allOr
newFilter.field = dataFilters.removeKeyNumbering(field)
return newFilter
})
}
onMount(() => {
parseFilters(filters)
rawFilters.forEach(filter => {
filter.type =
schemaFields.find(field => field.name === filter.field)?.type ||
filter.type
})
})
// Add field key prefixes and a special metadata filter object to indicate
// how to handle filter behaviour
const enrichFilters = rawFilters => {
let count = 1
return rawFilters
.filter(filter => filter.field)
.map(filter => ({
...filter,
field: `${count++}:${filter.field}`,
}))
.concat(...rawFilters.filter(filter => !filter.field))
}
</script>
<FilterBuilder
bind:filters={rawFilters}
behaviourFilters={true}
{schemaFields}
{datasource}
{allowBindings}
>
<div slot="filtering-hero-content" />
<DrawerBindableInput
let:filter
slot="binding"
disabled={filter.noValue}
title={filter.field}
value={filter.value}
placeholder="Value"
{panel}
{bindings}
on:change={event => {
const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id)
rawFilters[indexToUpdate] = {
...rawFilters[indexToUpdate],
value: event.detail,
}
}}
/>
</FilterBuilder>

27
packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
View File

@ -1,8 +1,14 @@
<script>
import { notifications, ActionButton, Button, Drawer } from "@budibase/bbui"
import {
notifications,
ActionButton,
Button,
Drawer,
DrawerContent,
} from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
import FilterDrawer from "./FilterDrawer.svelte"
import FilterBuilder from "./FilterBuilder.svelte"
import { selectedScreen } from "stores/builder"
const dispatch = createEventDispatcher()
@ -40,14 +46,15 @@
</div>
<Drawer bind:this={drawer} title="Filtering" on:drawerHide on:drawerShow>
<Button cta slot="buttons" on:click={saveFilter}>Save</Button>
<FilterDrawer
slot="body"
filters={value}
{bindings}
{schemaFields}
{datasource}
on:change={e => (tempValue = e.detail)}
/>
<DrawerContent slot="body">
<FilterBuilder
filters={value}
{bindings}
{schemaFields}
{datasource}
on:change={e => (tempValue = e.detail)}
/>
</DrawerContent>
</Drawer>
<style>

29
packages/builder/src/components/integration/KeyValueBuilder.svelte
View File

@ -35,6 +35,8 @@
export let bindingDrawerLeft
export let allowHelpers = true
export let customButtonText = null
export let keyBindings = false
export let allowJS = false
export let compare = (option, value) => option === value
let fields = Object.entries(object || {}).map(([name, value]) => ({
@ -116,12 +118,23 @@
class:readOnly-menu={readOnly && showMenu}
>
{#each fields as field, idx}
<Input
placeholder={keyPlaceholder}
readonly={readOnly}
bind:value={field.name}
on:blur={changed}
/>
{#if keyBindings}
<DrawerBindableInput
{bindings}
placeholder={keyPlaceholder}
on:blur={e => {
field.name = e.detail
changed()
}}
disabled={readOnly}
value={field.name}
{allowJS}
{allowHelpers}
drawerLeft={bindingDrawerLeft}
/>
{:else}
<Input readonly={readOnly} bind:value={field.name} on:blur={changed} />
{/if}
{#if isJsonArray(field.value)}
<Select readonly={true} value="Array" options={["Array"]} />
{:else if options}
@ -134,14 +147,14 @@
{:else if bindings && bindings.length}
<DrawerBindableInput
{bindings}
placeholder="Value"
placeholder={valuePlaceholder}
on:blur={e => {
field.value = e.detail
changed()
}}
disabled={readOnly}
value={field.value}
allowJS={false}
{allowJS}
{allowHelpers}
drawerLeft={bindingDrawerLeft}
/>

20
packages/builder/src/constants/backend/index.js
View File

@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = {
ONE: "One row",
}
export const ALLOWABLE_STRING_OPTIONS = [
FIELDS.STRING,
FIELDS.OPTIONS,
FIELDS.LONGFORM,
FIELDS.BARCODEQR,
]
export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
opt => opt.type
)
export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN]
export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map(
opt => opt.type
)
export const SWITCHABLE_TYPES = [
...ALLOWABLE_STRING_TYPES,
...ALLOWABLE_NUMBER_TYPES,
]
export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
export const BUDIBASE_DATASOURCE_TYPE = "budibase"

28
packages/builder/src/dataBinding.js
View File

@ -22,6 +22,7 @@ import {
isJSBinding,
decodeJSBinding,
encodeJSBinding,
getJsHelperList,
} from "@budibase/string-templates"
import { TableNames } from "./constants"
import { JSONUtils, Constants } from "@budibase/frontend-core"
@ -1210,9 +1211,32 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => {
if (!currentValue?.includes(from)) {
return false
}
if (convertTo === "readableBinding") {
// Dont replace if the value already matches the readable binding
// some cases we have the same binding for readable/runtime, specific logic for this
const sameBindings = binding.runtimeBinding.includes(binding.readableBinding)
const convertingToReadable = convertTo === "readableBinding"
const helperNames = Object.keys(getJsHelperList())
const matchedHelperNames = helperNames.filter(
name => name.includes(from) && currentValue.includes(name)
)
// edge case - if the binding is part of a helper it may accidentally replace it
if (matchedHelperNames.length > 0) {
const indexStart = currentValue.indexOf(from),
indexEnd = indexStart + from.length
for (let helperName of matchedHelperNames) {
const helperIndexStart = currentValue.indexOf(helperName),
helperIndexEnd = helperIndexStart + helperName.length
if (indexStart >= helperIndexStart && indexEnd <= helperIndexEnd) {
return false
}
}
}
if (convertingToReadable && !sameBindings) {
// Don't replace if the value already matches the readable binding
return currentValue.indexOf(binding.readableBinding) === -1
} else if (convertingToReadable) {
// if the runtime and readable bindings are very similar we have to assume it should be replaced
return true
}
// remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then
// this makes sure it is detected

2
packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js
View File

@ -4,8 +4,6 @@ import {
createDatasourceCreationStore,
} from "./datasourceCreation"
import { get } from "svelte/store"
// eslint-disable-next-line no-unused-vars
import { shouldIntegrationFetchTableNames } from "stores/selectors"
vi.mock("stores/selectors", () => ({
shouldIntegrationFetchTableNames: vi.fn(),

4
packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js
View File

@ -1,9 +1,9 @@
import { it, expect, describe, beforeEach, vi } from "vitest"
import { createOnGoogleAuthStore } from "./onGoogleAuth"
import { writable, get } from "svelte/store"
// eslint-disable-next-line no-unused-vars
// eslint-disable-next-line
import { params } from "@roxi/routify"
// eslint-disable-next-line no-unused-vars
// eslint-disable-next-line
import { integrations } from "stores/builder"
import { IntegrationTypes } from "constants/backend"

1
packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte
View File

@ -189,6 +189,7 @@
<Select
options={settingOptions}
bind:value={condition.setting}
on:change={() => delete condition.settingValue}
/>
<div>TO</div>
{#if definition}

10
packages/builder/src/stores/builder/tables.js
View File

@ -1,8 +1,8 @@
import { FieldType } from "@budibase/types"
import { SWITCHABLE_TYPES } from "@budibase/shared-core"
import { get, writable, derived } from "svelte/store"
import { cloneDeep } from "lodash/fp"
import { API } from "api"
import { SWITCHABLE_TYPES } from "constants/backend"
export function createTablesStore() {
const store = writable({
@ -64,7 +64,7 @@ export function createTablesStore() {
if (
oldField != null &&
oldField?.type !== field.type &&
SWITCHABLE_TYPES.indexOf(oldField?.type) === -1
!SWITCHABLE_TYPES[oldField?.type]?.includes(field.type)
) {
updatedTable.schema[key] = oldField
}
@ -148,12 +148,6 @@ export function createTablesStore() {
if (indexes) {
draft.indexes = indexes
}
// Add object to indicate if column is being added
if (draft.schema[field.name] === undefined) {
draft._add = {
name: field.name,
}
}
draft.schema = {
...draft.schema,
[field.name]: cloneDeep(field),

4
packages/cli/src/backups/objectStore.ts
View File

@ -9,7 +9,7 @@ const {
ObjectStore,
retrieve,
uploadDirectory,
makeSureBucketExists,
createBucketIfNotExists,
} = objectStore
const bucketList = Object.values(ObjectStoreBuckets)
@ -61,7 +61,7 @@ export async function importObjects() {
let count = 0
for (let bucket of buckets) {
const client = ObjectStore(bucket)
await makeSureBucketExists(client, bucket)
await createBucketIfNotExists(client, bucket)
const files = await uploadDirectory(bucket, join(path, bucket), "/")
count += files.length
bar.update(count)

7
packages/cli/src/hosting/utils.ts
View File

@ -54,11 +54,9 @@ export async function downloadDockerCompose() {
export async function checkDockerConfigured() {
const error =
"docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
"docker has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
const docker = await lookpath("docker")
const compose = await lookpath("docker-compose")
const composeV2 = await lookpath("docker compose")
if (!docker || (!compose && !composeV2)) {
if (!docker) {
throw error
}
}
@ -105,7 +103,6 @@ export function getAppService(path: string) {
}
export function updateDockerComposeService(
// eslint-disable-next-line no-unused-vars
updateFn: (service: DockerCompose) => void
) {
const opts = ["docker-compose.yaml", "docker-compose.yml"]

6
packages/client/manifest.json
View File

@ -6967,6 +6967,12 @@
"key": "stripeRows",
"defaultValue": false
},
{
"type": "boolean",
"label": "Quiet",
"key": "quiet",
"defaultValue": false
},
{
"section": true,
"name": "Columns",

240
packages/client/src/components/ClientApp.svelte
View File

@ -119,140 +119,142 @@
{/if}
</svelte:head>
<div
id="spectrum-root"
lang="en"
dir="ltr"
class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
class:builder={$builderStore.inBuilder}
class:show={fontsLoaded && dataLoaded}
>
{#if $environmentStore.maintenance.length > 0}
<MaintenanceScreen maintenanceList={$environmentStore.maintenance} />
{:else}
<DeviceBindingsProvider>
<UserBindingsProvider>
<StateBindingsProvider>
<RowSelectionProvider>
<QueryParamsProvider>
<SnippetsProvider>
<!-- Settings bar can be rendered outside of device preview -->
<!-- Key block needs to be outside the if statement or it breaks -->
{#key $builderStore.selectedComponentId}
{#if $builderStore.inBuilder}
<SettingsBar />
{/if}
{/key}
<!-- Clip boundary for selection indicators -->
<div
id="clip-root"
class:preview={$builderStore.inBuilder}
class:tablet-preview={$builderStore.previewDevice ===
"tablet"}
class:mobile-preview={$builderStore.previewDevice ===
"mobile"}
>
<!-- Actual app -->
<div id="app-root">
{#if showDevTools}
<DevToolsHeader />
{#if dataLoaded}
<div
id="spectrum-root"
lang="en"
dir="ltr"
class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
class:builder={$builderStore.inBuilder}
class:show={fontsLoaded && dataLoaded}
>
{#if $environmentStore.maintenance.length > 0}
<MaintenanceScreen maintenanceList={$environmentStore.maintenance} />
{:else}
<DeviceBindingsProvider>
<UserBindingsProvider>
<StateBindingsProvider>
<RowSelectionProvider>
<QueryParamsProvider>
<SnippetsProvider>
<!-- Settings bar can be rendered outside of device preview -->
<!-- Key block needs to be outside the if statement or it breaks -->
{#key $builderStore.selectedComponentId}
{#if $builderStore.inBuilder}
<SettingsBar />
{/if}
{/key}
<div id="app-body">
{#if permissionError}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
You don't have permission to use this app
</Heading>
<Body size="S">
Ask your administrator to grant you access
</Body>
</Layout>
</div>
{:else if !$screenStore.activeLayout}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
Something went wrong rendering your app
</Heading>
<Body size="S">
Get in touch with support if this issue persists
</Body>
</Layout>
</div>
{:else if embedNoScreens}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
This Budibase app is not publicly accessible
</Heading>
</Layout>
</div>
{:else}
<CustomThemeWrapper>
{#key $screenStore.activeLayout._id}
<Component
isLayout
instance={$screenStore.activeLayout.props}
/>
{/key}
<!-- Clip boundary for selection indicators -->
<div
id="clip-root"
class:preview={$builderStore.inBuilder}
class:tablet-preview={$builderStore.previewDevice ===
"tablet"}
class:mobile-preview={$builderStore.previewDevice ===
"mobile"}
>
<!-- Actual app -->
<div id="app-root">
{#if showDevTools}
<DevToolsHeader />
{/if}
<!--
<div id="app-body">
{#if permissionError}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
You don't have permission to use this app
</Heading>
<Body size="S">
Ask your administrator to grant you access
</Body>
</Layout>
</div>
{:else if !$screenStore.activeLayout}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
Something went wrong rendering your app
</Heading>
<Body size="S">
Get in touch with support if this issue persists
</Body>
</Layout>
</div>
{:else if embedNoScreens}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
This Budibase app is not publicly accessible
</Heading>
</Layout>
</div>
{:else}
<CustomThemeWrapper>
{#key $screenStore.activeLayout._id}
<Component
isLayout
instance={$screenStore.activeLayout.props}
/>
{/key}
<!--
Flatpickr needs to be inside the theme wrapper.
It also needs its own container because otherwise it hijacks
key events on the whole page. It is painful to work with.
-->
<div id="flatpickr-root" />
<div id="flatpickr-root" />
<!-- Modal container to ensure they sit on top -->
<div class="modal-container" />
<!-- Modal container to ensure they sit on top -->
<div class="modal-container" />
<!-- Layers on top of app -->
<NotificationDisplay />
<ConfirmationDisplay />
<PeekScreenDisplay />
</CustomThemeWrapper>
{/if}
<!-- Layers on top of app -->
<NotificationDisplay />
<ConfirmationDisplay />
<PeekScreenDisplay />
</CustomThemeWrapper>
{/if}
{#if showDevTools}
<DevTools />
{#if showDevTools}
<DevTools />
{/if}
</div>
{#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
<FreeFooter />
{/if}
</div>
{#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
<FreeFooter />
<!-- Preview and dev tools utilities -->
{#if $appStore.isDevApp}
<SelectionIndicator />
{/if}
{#if $builderStore.inBuilder || $devToolsStore.allowSelection}
<HoverIndicator />
{/if}
{#if $builderStore.inBuilder}
<DNDHandler />
<GridDNDHandler />
{/if}
</div>
<!-- Preview and dev tools utilities -->
{#if $appStore.isDevApp}
<SelectionIndicator />
{/if}
{#if $builderStore.inBuilder || $devToolsStore.allowSelection}
<HoverIndicator />
{/if}
{#if $builderStore.inBuilder}
<DNDHandler />
<GridDNDHandler />
{/if}
</div>
</SnippetsProvider>
</QueryParamsProvider>
</RowSelectionProvider>
</StateBindingsProvider>
</UserBindingsProvider>
</DeviceBindingsProvider>
{/if}
</div>
<KeyboardManager />
</SnippetsProvider>
</QueryParamsProvider>
</RowSelectionProvider>
</StateBindingsProvider>
</UserBindingsProvider>
</DeviceBindingsProvider>
{/if}
</div>
<KeyboardManager />
{/if}
<style>
#spectrum-root {

2
packages/client/src/components/app/GridBlock.svelte
View File

@ -11,6 +11,7 @@
export let allowEditRows = true
export let allowDeleteRows = true
export let stripeRows = false
export let quiet = false
export let initialFilter = null
export let initialSortColumn = null
export let initialSortOrder = null
@ -117,6 +118,7 @@
datasource={table}
{API}
{stripeRows}
{quiet}
{initialFilter}
{initialSortColumn}
{initialSortOrder}

5
packages/client/src/components/app/NavItem.svelte
View File

@ -157,6 +157,11 @@
width: 100%;
}
/* Use normal theme colors for links when using a top nav */
.dropdown:not(.left) .sublinks a {
color: var(--spectrum-alias-text-color);
}
/* Left dropdowns */
.dropdown.left .sublinks-wrapper {
display: none;

216
packages/client/src/components/app/dynamic-filter/FilterModal.svelte
View File

@ -1,216 +1,14 @@
<script>
import {
Body,
Button,
Combobox,
DatePicker,
Icon,
Input,
Layout,
Select,
} from "@budibase/bbui"
import { generate } from "shortid"
import { LuceneUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
import { FilterBuilder } from "@budibase/frontend-core"
export let schemaFields
export let filters = []
export let datasource
const context = getContext("context")
const BannedTypes = ["link", "attachment", "json"]
$: fieldOptions = (schemaFields ?? [])
.filter(
field =>
!BannedTypes.includes(field.type) ||
(field.type === "formula" && field.formulaType === "static")
)
.map(field => ({
label: field.displayName || field.name,
value: field.name,
}))
const addFilter = () => {
filters = [
...filters,
{
id: generate(),
field: null,
operator: Constants.OperatorOptions.Equals.value,
value: null,
valueType: "Value",
},
]
}
const removeFilter = id => {
filters = filters.filter(field => field.id !== id)
}
const duplicateFilter = id => {
const existingFilter = filters.find(filter => filter.id === id)
const duplicate = { ...existingFilter, id: generate() }
filters = [...filters, duplicate]
}
const onFieldChange = (expression, field) => {
// Update the field type
expression.type = schemaFields.find(x => x.name === field)?.type
expression.externalType = schemaFields.find(
x => x.name === field
)?.externalType
// Ensure a valid operator is set
const validOperators = LuceneUtils.getValidOperatorsForType(
{ type: expression.type },
expression.field,
datasource
).map(x => x.value)
if (!validOperators.includes(expression.operator)) {
expression.operator =
validOperators[0] ?? Constants.OperatorOptions.Equals.value
onOperatorChange(expression, expression.operator)
}
// if changed to an array, change default value to empty array
const idx = filters.findIndex(x => x.field === field)
if (expression.type === "array") {
filters[idx].value = []
} else {
filters[idx].value = null
}
}
const onOperatorChange = (expression, operator) => {
const noValueOptions = [
Constants.OperatorOptions.Empty.value,
Constants.OperatorOptions.NotEmpty.value,
]
expression.noValue = noValueOptions.includes(operator)
if (expression.noValue) {
expression.value = null
}
}
const getFieldOptions = field => {
const schema = schemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getSchema = filter => {
return schemaFields.find(field => field.name === filter.field)
}
</script>
<div class="container" class:mobile={$context.device.mobile}>
<Layout noPadding>
<Body size="S">
{#if !filters?.length}
Add your first filter expression.
{:else}
Results are filtered to only those which match all of the following
constraints.
{/if}
</Body>
{#if filters?.length}
<div class="fields">
{#each filters as filter}
<Select
bind:value={filter.field}
options={fieldOptions}
on:change={e => onFieldChange(filter, e.detail)}
placeholder="Column"
/>
<Select
disabled={!filter.field}
options={LuceneUtils.getValidOperatorsForType(
{ type: filter.type, subtype: filter.subtype },
filter.field,
datasource
)}
bind:value={filter.operator}
on:change={e => onOperatorChange(filter, e.detail)}
placeholder={null}
/>
{#if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "boolean"}
<Combobox
disabled={filter.noValue}
options={[
{ label: "True", value: "true" },
{ label: "False", value: "false" },
]}
bind:value={filter.value}
/>
{:else if filter.type === "datetime"}
<DatePicker
disabled={filter.noValue}
enableTime={!getSchema(filter).dateOnly}
timeOnly={getSchema(filter).timeOnly}
bind:value={filter.value}
/>
{:else}
<Input disabled />
{/if}
<div class="controls">
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
</div>
{/each}
</div>
{/if}
<div>
<Button icon="AddCircle" size="M" secondary on:click={addFilter}>
Add filter
</Button>
</div>
</Layout>
</div>
<style>
.container {
width: 100%;
max-width: 1000px;
margin: 0 auto;
}
.fields {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: 1fr 120px 1fr auto auto;
}
.controls {
display: contents;
}
.container.mobile .fields {
grid-template-columns: 1fr;
}
.container.mobile .controls {
display: flex;
flex-direction: row;
justify-content: flex-start;
align-items: center;
padding: var(--spacing-s) 0;
gap: var(--spacing-s);
}
</style>
<FilterBuilder bind:filters {schemaFields} {datasource} filtersLabel={null}>
<div slot="filtering-hero-content">
Results are filtered to only those which match all of the following
constraints.
</div>
</FilterBuilder>

12
packages/client/src/components/app/forms/AttachmentField.svelte
View File

@ -58,17 +58,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteAttachments({
keys: fileList,
tableId: formContext?.dataSource?.tableId,
})
} catch (error) {
return []
}
}
const handleChange = e => {
const value = fieldApiMapper.set(e.detail)
const changed = fieldApi.setValue(value)
@ -98,7 +87,6 @@
error={fieldState.error}
on:change={handleChange}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
{handleTooManyFiles}
{maximum}

1
packages/frontend-core/package.json
View File

@ -11,6 +11,7 @@
"@budibase/types": "0.0.0",
"dayjs": "^1.10.8",
"lodash": "4.17.21",
"shortid": "2.2.15",
"socket.io-client": "^4.6.1"
}
}

28
packages/frontend-core/src/api/attachments.js
View File

@ -61,34 +61,6 @@ export const buildAttachmentEndpoints = API => {
})
return { publicUrl }
},
/**
* Deletes attachments from the bucket.
* @param keys the attachments to delete
* @param tableId the associated table ID
*/
deleteAttachments: async ({ keys, tableId }) => {
return await API.post({
url: `/api/attachments/${tableId}/delete`,
body: {
keys,
},
})
},
/**
* Deletes attachments from the builder bucket.
* @param keys the attachments to delete
*/
deleteBuilderAttachments: async keys => {
return await API.post({
url: `/api/attachments/delete`,
body: {
keys,
},
})
},
/**
* Download an attachment from a row given its column name.
* @param datasourceId the ID of the datasource to download from

366
packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte → packages/frontend-core/src/components/FilterBuilder.svelte
View File

@ -4,33 +4,36 @@
Button,
Combobox,
DatePicker,
DrawerContent,
Icon,
Input,
Label,
Layout,
Multiselect,
Select,
Label,
Multiselect,
} from "@budibase/bbui"
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
import { FieldType, SearchFilterOperator } from "@budibase/types"
import { generate } from "shortid"
import { Constants, LuceneUtils } from "@budibase/frontend-core"
import { getFields } from "helpers/searchFields"
import { FieldType } from "@budibase/types"
import { createEventDispatcher, onMount } from "svelte"
import { LuceneUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
import FilterUsers from "./FilterUsers.svelte"
const { OperatorOptions } = Constants
export let schemaFields
export let filters = []
export let bindings = []
export let panel = ClientBindingPanel
export let allowBindings = true
export let datasource
export let behaviourFilters = false
export let allowBindings = false
export let filtersLabel = "Filters"
$: matchAny = filters?.find(filter => filter.operator === "allOr") != null
$: onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
$: fieldFilters = filters.filter(
filter => filter.operator !== "allOr" && !filter.onEmptyFilter
)
const dispatch = createEventDispatcher()
const { OperatorOptions } = Constants
const KeyedFieldRegex = /\d[0-9]*:/g
const behaviourOptions = [
{ value: "and", label: "Match all filters" },
{ value: "or", label: "Match any filter" },
@ -40,62 +43,18 @@
{ value: "none", label: "Return no rows" },
]
let rawFilters
let matchAny = false
let onEmptyFilter = "all"
const context = getContext("context")
$: parseFilters(filters)
$: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
matchAny = filters?.find(filter => filter.operator === "allOr") != null
onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
rawFilters = (filters || [])
.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
.map(filter => {
const { field } = filter
let newFilter = { ...filter }
delete newFilter.allOr
if (typeof field === "string" && field.match(KeyedFieldRegex) != null) {
const parts = field.split(":")
parts.shift()
newFilter.field = parts.join(":")
}
return newFilter
})
}
onMount(() => {
parseFilters(filters)
rawFilters.forEach(filter => {
filter.type =
schemaFields.find(field => field.name === filter.field)?.type ||
filter.type
})
})
// Add field key prefixes and a special metadata filter object to indicate
// how to handle filter behaviour
const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
let count = 1
return rawFilters
.filter(filter => filter.field)
.map(filter => ({
...filter,
field: `${count++}:${filter.field}`,
}))
.concat(matchAny ? [{ operator: "allOr" }] : [])
.concat([{ onEmptyFilter }])
}
$: fieldOptions = (schemaFields ?? [])
.filter(field => getValidOperatorsForType(field).length)
.map(field => ({
label: field.displayName || field.name,
value: field.name,
}))
const addFilter = () => {
rawFilters = [
...rawFilters,
filters = [
...(filters || []),
{
id: generate(),
field: null,
@ -107,22 +66,62 @@
}
const removeFilter = id => {
rawFilters = rawFilters.filter(field => field.id !== id)
filters = filters.filter(field => field.id !== id)
// Clear all filters when no fields are specified
if (filters.length === 1 && filters[0].onEmptyFilter) {
filters = []
}
}
const duplicateFilter = id => {
const existingFilter = rawFilters.find(filter => filter.id === id)
const existingFilter = filters.find(filter => filter.id === id)
const duplicate = { ...existingFilter, id: generate() }
rawFilters = [...rawFilters, duplicate]
filters = [...filters, duplicate]
}
const onFieldChange = filter => {
const previousType = filter.type
sanitizeTypes(filter)
sanitizeOperator(filter)
sanitizeValue(filter, previousType)
}
const onOperatorChange = filter => {
sanitizeOperator(filter)
sanitizeValue(filter, filter.type)
}
const onValueTypeChange = filter => {
sanitizeValue(filter)
}
const getFieldOptions = field => {
const schema = schemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getSchema = filter => {
return enrichedSchemaFields.find(field => field.name === filter.field)
return schemaFields.find(field => field.name === filter.field)
}
const getValidOperatorsForType = filter => {
if (!filter?.field && !filter?.name) {
return []
}
return LuceneUtils.getValidOperatorsForType(
filter,
filter.field || filter.name,
datasource
)
}
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
const sanitizeTypes = filter => {
// Update type based on field
const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
const fieldSchema = schemaFields.find(x => x.name === filter.field)
filter.type = fieldSchema?.type
filter.subtype = fieldSchema?.subtype
@ -154,88 +153,79 @@
// Ensure array values are properly set and cleared
if (Array.isArray(filter.value)) {
if (filter.valueType !== "Value" || filter.type !== "array") {
if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) {
filter.value = null
}
} else if (filter.type === "array" && filter.valueType === "Value") {
} else if (
filter.type === FieldType.ARRAY &&
filter.valueType === "Value"
) {
filter.value = []
} else if (
previousType !== filter.type &&
(previousType === FieldType.BB_REFERENCE ||
filter.type === FieldType.BB_REFERENCE)
) {
filter.value = filter.type === "array" ? [] : null
filter.value = filter.type === FieldType.ARRAY ? [] : null
}
}
const onFieldChange = filter => {
const previousType = filter.type
sanitizeTypes(filter)
sanitizeOperator(filter)
sanitizeValue(filter, previousType)
}
const onOperatorChange = filter => {
sanitizeOperator(filter)
sanitizeValue(filter, filter.type)
}
const onValueTypeChange = filter => {
sanitizeValue(filter)
}
const getFieldOptions = field => {
const schema = enrichedSchemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getValidOperatorsForType = filter => {
if (!filter?.field) {
return []
function handleAllOr(option) {
filters = filters.filter(f => f.operator !== "allOr")
if (option === "or") {
filters.push({ operator: "allOr" })
}
}
return LuceneUtils.getValidOperatorsForType(
{ type: filter.type, subtype: filter.subtype },
filter.field,
datasource
)
function handleOnEmptyFilter(value) {
filters = filters?.filter(filter => !filter.onEmptyFilter)
filters.push({ onEmptyFilter: value })
}
</script>
<DrawerContent>
<div class="container">
<Layout noPadding>
{#if !rawFilters?.length}
<Body size="S">Add your first filter expression.</Body>
{:else}
<div class="fields">
<Select
label="Behaviour"
value={matchAny ? "or" : "and"}
options={behaviourOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (matchAny = e.detail === "or")}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (onEmptyFilter = e.detail)}
placeholder={null}
/>
<div class="container" class:mobile={$context?.device?.mobile}>
<Layout noPadding>
{#if fieldOptions?.length}
<Body size="S">
{#if !fieldFilters?.length}
Add your first filter expression.
{:else}
<slot name="filtering-hero-content" />
{#if behaviourFilters}
<div class="behaviour-filters">
<Select
label="Behaviour"
value={matchAny ? "or" : "and"}
options={behaviourOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => handleAllOr(e.detail)}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => handleOnEmptyFilter(e.detail)}
placeholder={null}
/>
{/if}
</div>
{/if}
</div>
{/if}
</Body>
{#if fieldFilters?.length}
<div>
<div class="filter-label">
<Label>Filters</Label>
</div>
<div class="fields">
{#each rawFilters as filter}
{#if filtersLabel}
<div class="filter-label">
<Label>{filtersLabel}</Label>
</div>
{/if}
<div class="fields" class:with-bindings={allowBindings}>
{#each fieldFilters as filter}
<Select
bind:value={filter.field}
options={fieldOptions}
@ -249,38 +239,32 @@
on:change={() => onOperatorChange(filter)}
placeholder={null}
/>
<Select
disabled={filter.noValue || !filter.field}
options={valueTypeOptions}
bind:value={filter.valueType}
on:change={() => onValueTypeChange(filter)}
placeholder={null}
/>
{#if filter.field && filter.valueType === "Binding"}
<DrawerBindableInput
disabled={filter.noValue}
title={filter.field}
value={filter.value}
placeholder="Value"
{panel}
{bindings}
on:change={event => (filter.value = event.detail)}
{#if allowBindings}
<Select
disabled={filter.noValue || !filter.field}
options={valueTypeOptions}
bind:value={filter.valueType}
on:change={() => onValueTypeChange(filter)}
placeholder={null}
/>
{:else if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)}
{/if}
{#if allowBindings && filter.field && filter.valueType === "Binding"}
<slot name="binding" {filter} />
{:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")}
{:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)}
<Multiselect
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "options"}
{:else if filter.type === FieldType.OPTIONS}
<Combobox
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "boolean"}
{:else if filter.type === FieldType.BOOLEAN}
<Combobox
disabled={filter.noValue}
options={[
@ -289,7 +273,7 @@
]}
bind:value={filter.value}
/>
{:else if filter.type === "datetime"}
{:else if filter.type === FieldType.DATETIME}
<DatePicker
disabled={filter.noValue}
enableTime={!getSchema(filter)?.dateOnly}
@ -306,32 +290,36 @@
disabled={filter.noValue}
/>
{:else}
<DrawerBindableInput disabled />
<Input disabled />
{/if}
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
<div class="controls">
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
</div>
{/each}
</div>
</div>
{/if}
<div class="bottom">
<div>
<Button icon="AddCircle" size="M" secondary on:click={addFilter}>
Add filter
</Button>
</div>
</Layout>
</div>
</DrawerContent>
{:else}
<Body size="S">None of the table column can be used for filtering.</Body>
{/if}
</Layout>
</div>
<style>
.container {
@ -339,22 +327,42 @@
max-width: 1000px;
margin: 0 auto;
}
.fields {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: 1fr 120px 1fr auto auto;
}
.fields.with-bindings {
grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
}
.controls {
display: contents;
}
.container.mobile .fields {
grid-template-columns: 1fr;
}
.container.mobile .controls {
display: flex;
flex-direction: row;
justify-content: flex-start;
align-items: center;
padding: var(--spacing-s) 0;
gap: var(--spacing-s);
}
.filter-label {
margin-bottom: var(--spacing-s);
}
.bottom {
display: flex;
justify-content: space-between;
.behaviour-filters {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
}
</style>

4
packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte → packages/frontend-core/src/components/FilterUsers.svelte
View File

@ -1,9 +1,9 @@
<script>
import { Select, Multiselect } from "@budibase/bbui"
import { fetchData } from "@budibase/frontend-core"
import { createAPIClient } from "../api"
import { API } from "api"
export let API = createAPIClient()
export let value = null
export let disabled
export let multiselect = false

9
packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
View File

@ -61,14 +61,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
onMount(() => {
api = {
focus: () => open(),
@ -101,7 +93,6 @@
on:change={e => onChange(e.detail)}
maximum={maximum || schema.constraints?.length?.maximum}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>
</div>

8
packages/frontend-core/src/components/grid/layout/Grid.svelte
View File

@ -39,6 +39,7 @@
export let canEditColumns = true
export let canSaveSchema = true
export let stripeRows = false
export let quiet = false
export let collaboration = true
export let showAvatars = true
export let showControls = true
@ -91,6 +92,7 @@
canEditColumns,
canSaveSchema,
stripeRows,
quiet,
collaboration,
showAvatars,
showControls,
@ -124,6 +126,7 @@
class:is-resizing={$isResizing}
class:is-reordering={$isReordering}
class:stripe={stripeRows}
class:quiet
on:mouseenter={() => gridFocused.set(true)}
on:mouseleave={() => gridFocused.set(false)}
style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-height:{MaxCellRenderHeight}px; --max-cell-render-width-overflow:{MaxCellRenderWidthOverflow}px; --content-lines:{$contentLines};"
@ -331,4 +334,9 @@
.grid-data-outer :global(.spectrum-Checkbox-partialCheckmark) {
transition: none;
}
/* Overrides */
.grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)) {
border-right: none;
}
</style>

1
packages/frontend-core/src/components/index.js
View File

@ -6,3 +6,4 @@ export { default as UserAvatars } from "./UserAvatars.svelte"
export { default as Updating } from "./Updating.svelte"
export { Grid } from "./grid"
export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte"
export { default as FilterBuilder } from "./FilterBuilder.svelte"

3
packages/frontend-core/src/fetch/DataFetch.js
View File

@ -348,8 +348,7 @@ export default class DataFetch {
* Determine the feature flag for this datasource definition
* @param definition
*/
// eslint-disable-next-line no-unused-vars
determineFeatureFlags(definition) {
determineFeatureFlags(_definition) {
return {
supportsSearch: false,
supportsSort: false,

2
packages/pro

@ -1 +1 @@
Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a
Subproject commit dff7b5a9dd1fd770f8a48fb8e6df1740be605f18

5
packages/server/package.json
View File

@ -61,14 +61,17 @@
"@google-cloud/firestore": "6.8.0",
"@koa/router": "8.0.8",
"@socket.io/redis-adapter": "^8.2.1",
"@types/xml2js": "^0.4.14",
"airtable": "0.10.1",
"arangojs": "7.2.0",
"archiver": "7.0.1",
"aws-sdk": "2.1030.0",
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bl": "^6.0.12",
"bull": "4.10.1",
"chokidar": "3.5.3",
"content-disposition": "^0.5.4",
"cookies": "0.8.0",
"csvtojson": "2.0.10",
"curlconverter": "3.21.0",
@ -125,7 +128,7 @@
"@babel/preset-env": "7.16.11",
"@swc/core": "1.3.71",
"@swc/jest": "0.2.27",
"@types/archiver": "^6.0.2",
"@types/archiver": "6.0.2",
"@types/global-agent": "2.1.1",
"@types/google-spreadsheet": "3.1.5",
"@types/jest": "29.5.5",

1
packages/server/scripts/integrations/mssql/docker-compose.yaml
View File

@ -4,6 +4,7 @@ services:
# user: sa
# database: master
mssql:
# platform: linux/amd64
image: bb/mssql
build:
context: .

2
packages/server/specs/generate.ts
View File

@ -76,7 +76,7 @@ function writeFile(output: any, filename: string) {
console.log(`Wrote spec to ${path}`)
return path
} catch (err) {
console.error(err)
console.error("Error writing spec file", err)
}
}

42
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { db as dbCore } from "@budibase/backend-core"
import AliasTables from "./alias"
import sdk from "../../../sdk"
import env from "../../../environment"
@ -120,6 +119,9 @@ async function removeManyToManyRelationships(
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
},
})
} else {
return []
@ -134,6 +136,9 @@ async function removeOneToManyRelationships(rowId: string, table: Table) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
filters,
meta: {
table,
},
})
} else {
return []
@ -249,6 +254,9 @@ export class ExternalRequest<T extends Operation> {
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(table._id!, Operation.READ),
filters: buildFilters(rowId, {}, table),
meta: {
table,
},
})
if (Array.isArray(response) && response.length > 0) {
return response[0]
@ -366,36 +374,45 @@ export class ExternalRequest<T extends Operation> {
) {
continue
}
let tableId: string | undefined,
let relatedTableId: string | undefined,
lookupField: string | undefined,
fieldName: string | undefined
if (isManyToMany(field)) {
tableId = field.through
relatedTableId = field.through
lookupField = primaryKey
fieldName = field.throughTo || primaryKey
} else if (isManyToOne(field)) {
tableId = field.tableId
relatedTableId = field.tableId
lookupField = field.foreignKey
fieldName = field.fieldName
}
if (!tableId || !lookupField || !fieldName) {
if (!relatedTableId || !lookupField || !fieldName) {
throw new Error(
"Unable to lookup relationships - undefined column properties."
)
}
const { tableName: relatedTableName } = breakExternalTableId(tableId)
const { tableName: relatedTableName } =
breakExternalTableId(relatedTableId)
// @ts-ignore
const linkPrimaryKey = this.tables[relatedTableName].primary[0]
if (!lookupField || !row[lookupField]) {
continue
}
const endpoint = getEndpoint(relatedTableId, Operation.READ)
const relatedTable = this.tables[endpoint.entityId]
if (!relatedTable) {
throw new Error("unable to find related table")
}
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.READ),
endpoint: endpoint,
filters: {
equal: {
[fieldName]: row[lookupField],
},
},
meta: {
table: relatedTable,
},
})
// this is the response from knex if no rows found
const rows: Row[] =
@ -403,7 +420,11 @@ export class ExternalRequest<T extends Operation> {
const storeTo = isManyToMany(field)
? field.throughFrom || linkPrimaryKey
: fieldName
related[storeTo] = { rows, isMany: isManyToMany(field), tableId }
related[storeTo] = {
rows,
isMany: isManyToMany(field),
tableId: relatedTableId,
}
}
return related
}
@ -471,6 +492,9 @@ export class ExternalRequest<T extends Operation> {
// if we're doing many relationships then we're writing, only one response
body,
filters: buildFilters(id, {}, linkTable),
meta: {
table: linkTable,
},
})
)
} else {
@ -618,7 +642,7 @@ export class ExternalRequest<T extends Operation> {
if (env.SQL_ALIASING_DISABLE) {
response = await getDatasourceAndQuery(json)
} else {
const aliasing = new AliasTables(Object.keys(this.tables))
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
response = await aliasing.queryWithAliasing(json)
}

9
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -62,12 +62,12 @@ export function basicProcessing({
row,
table,
isLinked,
internal,
sqs,
}: {
row: Row
table: Table
isLinked: boolean
internal?: boolean
sqs?: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
@ -84,12 +84,13 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
if (!internal) {
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
} else {
for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
const columns = Object.keys(table.schema)
for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,

4
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
continue
}
let linked = await basicProcessing({
let linked = basicProcessing({
row,
table: linkedTable,
isLinked: true,
internal: opts?.sqs,
sqs: opts?.sqs,
})
if (!linked._id) {
continue

6
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
let rowId = row._id
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
row._id = rowId
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
row,
table,
isLinked: false,
internal: opts?.sqs,
sqs: opts?.sqs,
}),
table
)
@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
tables,
row,
finalRows,
relationships
relationships,
opts
)
}

7
packages/server/src/api/controllers/static/index.ts
View File

@ -127,13 +127,6 @@ export const uploadFile = async function (
)
}
export const deleteObjects = async function (ctx: Ctx) {
ctx.body = await objectStore.deleteFiles(
ObjectStoreBuckets.APPS,
ctx.request.body.keys
)
}
const requiresMigration = async (ctx: Ctx) => {
const appId = context.getAppId()
if (!appId) {

1
packages/server/src/api/controllers/table/ExternalRequest.ts
View File

@ -22,6 +22,7 @@ export async function makeTableRequest(
operation,
},
meta: {
table,
tables,
},
table,

3
packages/server/src/api/controllers/table/external.ts
View File

@ -31,7 +31,6 @@ export async function save(
renaming?: RenameColumn
) {
const inputs = ctx.request.body
const adding = inputs?._add
// can't do this right now
delete inputs.rows
const tableId = ctx.request.body._id
@ -44,7 +43,7 @@ export async function save(
const { datasource, table } = await sdk.tables.external.save(
datasourceId!,
inputs,
{ tableId, renaming, adding }
{ tableId, renaming }
)
builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table

5
packages/server/src/api/controllers/table/index.ts
View File

@ -77,11 +77,6 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const renaming = ctx.request.body._rename
const api = pickApi({ table })
// do not pass _rename or _add if saving to CouchDB
if (api === internal) {
delete ctx.request.body._add
delete ctx.request.body._rename
}
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
savedTable = sdk.tables.enrichViewSchemas(savedTable)

2
packages/server/src/api/controllers/table/internal.ts
View File

@ -16,7 +16,7 @@ export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
const { rows, ...rest } = ctx.request.body
const { _rename, rows, ...rest } = ctx.request.body
let tableToSave: Table = {
_id: generateTableID(),
...rest,

11
packages/server/src/api/routes/static.ts
View File

@ -32,11 +32,6 @@ router
.get("/builder/:file*", controller.serveBuilder)
.get("/api/assets/client", controller.serveClientLibrary)
.post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
.post(
"/api/attachments/delete",
authorized(BUILDER),
controller.deleteObjects
)
.post("/api/beta/:feature", controller.toggleBetaUiFeature)
.post(
"/api/attachments/:tableId/upload",
@ -44,12 +39,6 @@ router
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.uploadFile
)
.post(
"/api/attachments/:tableId/delete",
paramResource("tableId"),
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.deleteObjects
)
.get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
.get("/app/:appUrl/:path*", controller.serveApp)
.get("/:appId/:path*", controller.serveApp)

162
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -6,7 +6,19 @@ import sdk from "../../../sdk"
import tk from "timekeeper"
import { mocks } from "@budibase/backend-core/tests"
import { QueryPreview, SourceName } from "@budibase/types"
import {
Datasource,
FieldSchema,
FieldSubtype,
FieldType,
QueryPreview,
RelationshipType,
SourceName,
Table,
TableSchema,
} from "@budibase/types"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
tk.freeze(mocks.date.MOCK_DATE)
@ -223,4 +235,152 @@ describe("/datasources", () => {
})
})
})
describe.each([
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("fetch schema (%s)", (_, dsProvider) => {
beforeAll(async () => {
datasource = await config.api.datasource.create(await dsProvider)
})
it("fetching schema will not drop tables or columns", async () => {
const datasourceId = datasource!._id!
const simpleTable = await config.api.table.save(
tableForDatasource(datasource, {
name: "simple",
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
type SupportedSqlTypes =
| FieldType.STRING
| FieldType.BARCODEQR
| FieldType.LONGFORM
| FieldType.OPTIONS
| FieldType.DATETIME
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.FORMULA
| FieldType.BIGINT
| FieldType.BB_REFERENCE
| FieldType.LINK
| FieldType.ARRAY
const fullSchema: {
[type in SupportedSqlTypes]: FieldSchema & { type: type }
} = {
[FieldType.STRING]: {
name: "string",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
[FieldType.LONGFORM]: {
name: "longform",
type: FieldType.LONGFORM,
},
[FieldType.OPTIONS]: {
name: "options",
type: FieldType.OPTIONS,
constraints: {
presence: { allowEmpty: false },
},
},
[FieldType.NUMBER]: {
name: "number",
type: FieldType.NUMBER,
},
[FieldType.BOOLEAN]: {
name: "boolean",
type: FieldType.BOOLEAN,
},
[FieldType.ARRAY]: {
name: "array",
type: FieldType.ARRAY,
},
[FieldType.DATETIME]: {
name: "datetime",
type: FieldType.DATETIME,
dateOnly: true,
timeOnly: false,
},
[FieldType.LINK]: {
name: "link",
type: FieldType.LINK,
tableId: simpleTable._id!,
relationshipType: RelationshipType.ONE_TO_MANY,
fieldName: "link",
},
[FieldType.FORMULA]: {
name: "formula",
type: FieldType.FORMULA,
formula: "any formula",
},
[FieldType.BARCODEQR]: {
name: "barcodeqr",
type: FieldType.BARCODEQR,
},
[FieldType.BIGINT]: {
name: "bigint",
type: FieldType.BIGINT,
},
[FieldType.BB_REFERENCE]: {
name: "bb_reference",
type: FieldType.BB_REFERENCE,
subtype: FieldSubtype.USERS,
},
}
await config.api.table.save(
tableForDatasource(datasource, {
name: "full",
schema: fullSchema,
})
)
const persisted = await config.api.datasource.get(datasourceId)
await config.api.datasource.fetchSchema(datasourceId)
const updated = await config.api.datasource.get(datasourceId)
const expected: Datasource = {
...persisted,
entities:
persisted?.entities &&
Object.entries(persisted.entities).reduce<Record<string, Table>>(
(acc, [tableName, table]) => {
acc[tableName] = {
...table,
primaryDisplay: expect.not.stringMatching(
new RegExp(`^${table.primaryDisplay || ""}$`)
),
schema: Object.entries(table.schema).reduce<TableSchema>(
(acc, [fieldName, field]) => {
acc[fieldName] = expect.objectContaining({
...field,
})
return acc
},
{}
),
}
return acc
},
{}
),
_rev: expect.any(String),
}
expect(updated).toEqual(expected)
})
})
})

18
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -4,6 +4,7 @@ import {
Query,
QueryPreview,
SourceName,
TableSourceType,
} from "@budibase/types"
import * as setup from "../utilities"
import {
@ -740,12 +741,25 @@ describe.each(
})
describe("query through datasource", () => {
it("should be able to query a pg datasource", async () => {
it("should be able to query the datasource", async () => {
const entityId = "test_table"
await config.api.datasource.update({
...datasource,
entities: {
[entityId]: {
name: entityId,
schema: {},
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
},
},
})
const res = await config.api.datasource.query({
endpoint: {
datasourceId: datasource._id!,
operation: Operation.READ,
entityId: "test_table",
entityId,
},
resource: {
fields: ["id", "name"],

125
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -26,6 +26,7 @@ describe.each([
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/api/:sourceId/search (%s)", (name, dsProvider) => {
const isSqs = name === "internal-sqs"
const isInternal = name === "internal"
const config = setup.getConfig()
let envCleanup: (() => void) | undefined
@ -336,6 +337,20 @@ describe.each([
expectQuery({
range: { age: { low: 5, high: 9 } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { age: { low: 5 } },
}).toContainExactly([{ age: 10 }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { age: { high: 5 } },
}).toContainExactly([{ age: 1 }]))
})
describe("sort", () => {
@ -440,6 +455,20 @@ describe.each([
expectQuery({
range: { dob: { low: JAN_5TH, high: JAN_9TH } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { dob: { low: JAN_5TH } },
}).toContainExactly([{ dob: JAN_10TH }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { dob: { high: JAN_5TH } },
}).toContainExactly([{ dob: JAN_1ST }]))
})
describe("sort", () => {
@ -550,4 +579,100 @@ describe.each([
]))
})
})
describe("bigints", () => {
const SMALL = "1"
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
beforeAll(async () => {
await createTable({
num: { name: "num", type: FieldType.BIGINT },
})
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
})
describe("equal", () => {
it("successfully finds a row", () =>
expectQuery({ equal: { num: SMALL } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds a big value", () =>
expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }]))
it("fails to find nonexistent row", () =>
expectQuery({ equal: { num: "2" } }).toFindNothing())
})
describe("notEqual", () => {
it("successfully finds a row", () =>
expectQuery({ notEqual: { num: SMALL } }).toContainExactly([
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ notEqual: { num: 10 } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
})
describe("oneOf", () => {
it("successfully finds a row", () =>
expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds all rows", () =>
expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ oneOf: { num: [2] } }).toFindNothing())
})
// Range searches against bigints don't seem to work at all in Lucene, and I
// couldn't figure out why. Given that we're replacing Lucene with SQS,
// we've decided not to spend time on it.
!isInternal &&
describe("range", () => {
it("successfully finds a row", () =>
expectQuery({
range: { num: { low: SMALL, high: "5" } },
}).toContainExactly([{ num: SMALL }]))
it("successfully finds multiple rows", () =>
expectQuery({
range: { num: { low: SMALL, high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
it("successfully finds a row with a high bound", () =>
expectQuery({
range: { num: { low: MEDIUM, high: BIG } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("successfully finds no rows", () =>
expectQuery({
range: { num: { low: "5", high: "5" } },
}).toFindNothing())
it("can search using just a low value", () =>
expectQuery({
range: { num: { low: MEDIUM } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("can search using just a high value", () =>
expectQuery({
range: { num: { high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
})
})
})

9
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -34,7 +34,7 @@ describe.each([
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/tables (%s)", (_, dsProvider) => {
let isInternal: boolean
const isInternal: boolean = !dsProvider
let datasource: Datasource | undefined
let config = setup.getConfig()
@ -44,9 +44,6 @@ describe.each([
await config.init()
if (dsProvider) {
datasource = await config.api.datasource.create(await dsProvider)
isInternal = false
} else {
isInternal = true
}
})
@ -219,9 +216,6 @@ describe.each([
it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = {
_add: {
name: "NEW_COLUMN",
},
...basicTable(),
}
@ -235,7 +229,6 @@ describe.each([
updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
views: {},
}
delete expectedResponse._add
expect(response).toEqual(expectedResponse)
})
})

12
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -8,7 +8,7 @@ import {
PermissionLevel,
QuotaUsageType,
SaveTableRequest,
SearchQueryOperators,
SearchFilterOperator,
SortOrder,
SortType,
StaticQuotaName,
@ -132,7 +132,7 @@ describe.each([
primaryDisplay: generator.word(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "field",
value: "value",
},
@ -236,7 +236,7 @@ describe.each([
...view,
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -263,7 +263,7 @@ describe.each([
primaryDisplay: generator.word(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: generator.word(),
value: generator.word(),
},
@ -341,7 +341,7 @@ describe.each([
tableId: generator.guid(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -671,7 +671,7 @@ describe.each([
name: generator.guid(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "two",
value: "bar2",
},

12
packages/server/src/automations/steps/sendSmtpEmail.ts
View File

@ -7,6 +7,7 @@ import {
AutomationStepType,
AutomationIOType,
AutomationFeature,
AutomationCustomIOType,
} from "@budibase/types"
export const definition: AutomationStepSchema = {
@ -72,10 +73,10 @@ export const definition: AutomationStepSchema = {
title: "Location",
dependsOn: "addInvite",
},
url: {
type: AutomationIOType.STRING,
title: "URL",
dependsOn: "addInvite",
attachments: {
type: AutomationIOType.ATTACHMENT,
customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
title: "Attachments",
},
},
required: ["to", "from", "subject", "contents"],
@ -110,11 +111,13 @@ export async function run({ inputs }: AutomationStepInput) {
summary,
location,
url,
attachments,
} = inputs
if (!contents) {
contents = "<h1>No content</h1>"
}
to = to || undefined
try {
let response = await sendSmtpEmail({
to,
@ -124,6 +127,7 @@ export async function run({ inputs }: AutomationStepInput) {
cc,
bcc,
automation: true,
attachments,
invite: addInvite
? {
startTime,

1
packages/server/src/automations/steps/triggerAutomationRun.ts
View File

@ -62,7 +62,6 @@ export const definition: AutomationStepSchema = {
}
export async function run({ inputs }: AutomationStepInput) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { automationId, ...fieldParams } = inputs.automation
if (await features.isTriggerAutomationRunEnabled()) {

8
packages/server/src/automations/tests/sendSmtpEmail.spec.ts
View File

@ -50,6 +50,10 @@ describe("test the outgoing webhook action", () => {
cc: "cc",
bcc: "bcc",
addInvite: true,
attachments: [
{ url: "attachment1", filename: "attachment1.txt" },
{ url: "attachment2", filename: "attachment2.txt" },
],
...invite,
}
let resp = generateResponse(inputs.to, inputs.from)
@ -69,6 +73,10 @@ describe("test the outgoing webhook action", () => {
bcc: "bcc",
invite,
automation: true,
attachments: [
{ url: "attachment1", filename: "attachment1.txt" },
{ url: "attachment2", filename: "attachment2.txt" },
],
})
})
})

1
packages/server/src/db/utils.ts
View File

@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
export const SQS_DATASOURCE_INTERNAL = "internal"
export const ViewName = dbCore.ViewName
export const InternalTables = dbCore.InternalTable
export const UNICODE_MAX = dbCore.UNICODE_MAX

2
packages/server/src/environment.ts
View File

@ -28,6 +28,7 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@ -39,6 +40,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,

3
packages/server/src/features.ts
View File

@ -1,11 +1,8 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
// eslint-disable-next-line no-unused-vars
enum AppFeature {
// eslint-disable-next-line no-unused-vars
API = "api",
// eslint-disable-next-line no-unused-vars
AUTOMATIONS = "automations",
}

67
packages/server/src/integration-test/mysql.spec.ts
View File

@ -16,7 +16,6 @@ import {
getDatasource,
rawQuery,
} from "../integrations/tests/utils"
import { builderSocket } from "../websockets"
import { generator } from "@budibase/backend-core/tests"
// @ts-ignore
fetch.mockSearch()
@ -233,72 +232,6 @@ describe("mysql integrations", () => {
})
describe("POST /api/tables/", () => {
const emitDatasourceUpdateMock = jest.fn()
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: uniqueTableName(),
sourceId: datasource._id!,
primary: ["id"],
schema: {
id: {
type: FieldType.AUTO,
name: "id",
autocolumn: true,
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
},
},
_add: {
name: "new_column",
},
}
jest
.spyOn(builderSocket!, "emitDatasourceUpdate")
.mockImplementation(emitDatasourceUpdateMock)
await makeRequest("post", "/api/tables/", addColumnToTable)
const expectedTable: TableRequest = {
...addColumnToTable,
schema: {
id: {
type: FieldType.NUMBER,
name: "id",
autocolumn: true,
constraints: {
presence: false,
},
externalType: "int unsigned",
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
autocolumn: false,
constraints: {
presence: false,
},
externalType: "float(8,2)",
},
},
created: true,
_id: `${datasource._id}__${addColumnToTable.name}`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities![expectedTable.name]).toEqual(
expectedTable
)
})
it("will rename a column", async () => {
await makeRequest("post", "/api/tables/", primaryMySqlTable)

12
packages/server/src/integrations/base/query.ts
View File

@ -2,6 +2,7 @@ import {
QueryJson,
Datasource,
DatasourcePlusQueryResponse,
RowOperations,
} from "@budibase/types"
import { getIntegration } from "../index"
import sdk from "../../sdk"
@ -10,6 +11,17 @@ export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
): Promise<DatasourcePlusQueryResponse> {
const entityId = json.endpoint.entityId,
tableName = json.meta.table.name,
tableId = json.meta.table._id
// case found during testing - make sure this doesn't happen again
if (
RowOperations.includes(json.endpoint.operation) &&
entityId !== tableId &&
entityId !== tableName
) {
throw new Error("Entity ID and table metadata do not align")
}
datasource = await sdk.datasources.enrich(datasource)
const Integration = await getIntegration(datasource.source)
// query is the opinionated function

86
packages/server/src/integrations/base/sql.ts
View File

@ -6,6 +6,7 @@ import {
SqlClient,
isValidFilter,
getNativeSql,
SqlStatements,
} from "../utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
@ -22,6 +23,8 @@ import {
SortDirection,
SqlQueryBinding,
Table,
TableSourceType,
INTERNAL_TABLE_SOURCE_ID,
} from "@budibase/types"
import environment from "../../environment"
@ -135,6 +138,18 @@ function generateSelectStatement(
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
return table?._id
} else {
return table?.name
}
}
class InternalBuilder {
private readonly client: string
@ -146,10 +161,20 @@ class InternalBuilder {
addFilters(
query: Knex.QueryBuilder,
filters: SearchFilters | undefined,
tableName: string,
table: Table,
opts: { aliases?: Record<string, string>; relationship?: boolean }
): Knex.QueryBuilder {
function getTableName(name: string) {
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
const sqlStatements = new SqlStatements(this.client, table, { allOr })
const tableName =
this.client === SqlClient.SQL_LITE ? table._id! : table.name
function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
return alias || name
}
@ -161,11 +186,11 @@ class InternalBuilder {
const updatedKey = dbCore.removeKeyNumbering(key)
const isRelationshipField = updatedKey.includes(".")
if (!opts.relationship && !isRelationshipField) {
fn(`${getTableName(tableName)}.${updatedKey}`, value)
fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
}
if (opts.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
fn(`${getTableName(filterTableName)}.${property}`, value)
fn(`${getTableAlias(filterTableName)}.${property}`, value)
}
}
}
@ -244,12 +269,6 @@ class InternalBuilder {
}
}
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
if (filters.oneOf) {
iterate(filters.oneOf, (key, array) => {
const fnc = allOr ? "orWhereIn" : "whereIn"
@ -292,17 +311,11 @@ class InternalBuilder {
const lowValid = isValidFilter(value.low),
highValid = isValidFilter(value.high)
if (lowValid && highValid) {
// Use a between operator if we have 2 valid range values
const fnc = allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [value.low, value.high])
query = sqlStatements.between(query, key, value.low, value.high)
} else if (lowValid) {
// Use just a single greater than operator if we only have a low
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, ">", value.low)
query = sqlStatements.lte(query, key, value.low)
} else if (highValid) {
// Use just a single less than operator if we only have a high
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, "<", value.high)
query = sqlStatements.gte(query, key, value.high)
}
})
}
@ -345,10 +358,11 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
const table = json.meta.table
const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
@ -458,14 +472,13 @@ class InternalBuilder {
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}
const query = knex(
this.tableNameWithSchema(tableName, {
alias: tableAlias,
schema: endpoint.schema,
})
)
return query
}
@ -532,7 +545,7 @@ class InternalBuilder {
if (foundOffset) {
query = query.offset(foundOffset)
}
query = this.addFilters(query, filters, tableName, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// add sorting to pre-query
@ -553,7 +566,7 @@ class InternalBuilder {
endpoint.schema,
tableAliases
)
return this.addFilters(query, filters, tableName, {
return this.addFilters(query, filters, json.meta.table, {
relationship: true,
aliases: tableAliases,
})
@ -563,7 +576,7 @@ class InternalBuilder {
const { endpoint, body, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
const parsedBody = parseBody(body)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -577,7 +590,7 @@ class InternalBuilder {
delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -669,7 +682,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
// when creating if an ID has been inserted need to make sure
// the id filter is enriched with it before trying to retrieve the row
checkLookupKeys(id: any, json: QueryJson) {
if (!id || !json.meta?.table || !json.meta.table.primary) {
if (!id || !json.meta.table || !json.meta.table.primary) {
return json
}
const primaryKey = json.meta.table.primary?.[0]
@ -729,12 +742,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
results: Record<string, any>[],
aliases?: Record<string, string>
): Record<string, any>[] {
const tableName = getTableName(table)
for (const [name, field] of Object.entries(table.schema)) {
if (!this._isJsonColumn(field)) {
continue
}
const tableName = aliases?.[table.name] || table.name
const fullName = `${tableName}.${name}`
const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
const fullName = `${aliasedTableName}.${name}`
for (let row of results) {
if (typeof row[fullName] === "string") {
row[fullName] = JSON.parse(row[fullName])

23
packages/server/src/integrations/base/sqlTable.ts
View File

@ -1,19 +1,20 @@
import { Knex, knex } from "knex"
import {
RelationshipType,
FieldSubtype,
FieldType,
NumberFieldMetadata,
Operation,
QueryJson,
RelationshipType,
RenameColumn,
Table,
FieldType,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
import { utils } from "@budibase/shared-core"
import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
import { utils } from "@budibase/shared-core"
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
@ -105,13 +106,13 @@ function generateSchema(
column.relationshipType !== RelationshipType.MANY_TO_MANY
) {
if (!column.foreignKey || !column.tableId) {
throw "Invalid relationship schema"
throw new Error("Invalid relationship schema")
}
const { tableName } = breakExternalTableId(column.tableId)
// @ts-ignore
const relatedTable = tables[tableName]
if (!relatedTable) {
throw "Referenced table doesn't exist"
throw new Error("Referenced table doesn't exist")
}
const relatedPrimary = relatedTable.primary[0]
const externalType = relatedTable.schema[relatedPrimary].externalType
@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
let query: Knex.SchemaBuilder
if (!json.table || !json.meta || !json.meta.tables) {
throw "Cannot execute without table being specified"
throw new Error("Cannot execute without table being specified")
}
if (json.table.sourceType === TableSourceType.INTERNAL) {
throw new Error("Cannot perform table actions for SQS.")
}
switch (this._operation(json)) {
case Operation.CREATE_TABLE:
query = buildCreateTable(client, json.table, json.meta.tables)
break
case Operation.UPDATE_TABLE:
if (!json.meta || !json.meta.table) {
throw "Must specify old table for update"
throw new Error("Must specify old table for update")
}
// renameColumn does not work for MySQL, so return a raw query
if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
query = buildDeleteTable(client, json.table)
break
default:
throw "Table operation is of unknown type"
throw new Error("Table operation is of unknown type")
}
return getNativeSql(query)
}

3
packages/server/src/integrations/googlesheets.ts
View File

@ -168,8 +168,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return ""
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getStringConcat(parts: string[]) {
getStringConcat(_parts: string[]) {
return ""
}

2
packages/server/src/integrations/oracle.ts
View File

@ -378,7 +378,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
try {
await connection.close()
} catch (err) {
console.error(err)
console.error("Error connecting to Oracle", err)
}
}
}

68
packages/server/src/integrations/rest.ts
View File

@ -21,6 +21,10 @@ import { performance } from "perf_hooks"
import FormData from "form-data"
import { URLSearchParams } from "url"
import { blacklist } from "@budibase/backend-core"
import { handleFileResponse, handleXml } from "./utils"
import { parse } from "content-disposition"
import path from "path"
import { Builder as XmlBuilder } from "xml2js"
const BodyTypes = {
NONE: "none",
@ -57,8 +61,6 @@ const coreFields = {
},
}
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
description:
@ -129,42 +131,44 @@ class RestIntegration implements IntegrationBase {
}
async parseResponse(response: any, pagination: PaginationConfig | null) {
let data, raw, headers
let data, raw, headers, filename
const contentType = response.headers.get("content-type") || ""
const contentDisposition = response.headers.get("content-disposition") || ""
if (
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")
) {
filename =
path.basename(parse(contentDisposition).parameters?.filename) || ""
}
try {
if (response.status === 204) {
data = []
raw = []
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
const rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
raw = rawXml
} else if (contentType.includes("application/pdf")) {
data = await response.arrayBuffer() // Save PDF as ArrayBuffer
raw = Buffer.from(data)
if (filename) {
return handleFileResponse(response, filename, this.startTimeMs)
} else {
data = await response.text()
raw = data
if (response.status === 204) {
data = []
raw = []
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
let xmlResponse = await handleXml(response)
data = xmlResponse.data
raw = xmlResponse.rawXml
} else {
data = await response.text()
raw = data
}
}
} catch (err) {
throw "Failed to parse response body."
throw `Failed to parse response body: ${err}`
}
const size = formatBytes(
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
)

116
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -13,9 +13,23 @@ jest.mock("node-fetch", () => {
}))
})
import fetch from "node-fetch"
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
return {
...core,
context: {
...core.context,
getProdAppId: jest.fn(() => "app-id"),
},
}
})
jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
import { default as RestIntegration } from "../rest"
import { RestAuthType } from "@budibase/types"
import fetch from "node-fetch"
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
import { Readable } from "stream"
const FormData = require("form-data")
const { URLSearchParams } = require("url")
@ -611,4 +625,104 @@ describe("REST Integration", () => {
expect(calledConfig.headers).toEqual({})
expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
})
describe("File Handling", () => {
beforeAll(async () => {
jest.unmock("aws-sdk")
await objectStoreTestProviders.minio.start()
})
afterAll(async () => {
await objectStoreTestProviders.minio.stop()
})
it("uploads file to object store and returns signed URL", async () => {
const responseData = Buffer.from("teest file contnt")
const filename = "test.tar.gz"
const contentType = "application/gzip"
const mockReadable = new Readable()
mockReadable.push(responseData)
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
headers: {
raw: () => ({
"content-type": [contentType],
"content-disposition": [`attachment; filename="${filename}"`],
}),
get: (header: any) => {
if (header === "content-type") return contentType
if (header === "content-disposition")
return `attachment; filename="${filename}"`
},
},
body: mockReadable,
})
)
const query = {
path: "api",
}
const response = await config.integration.read(query)
expect(response.data).toEqual({
size: responseData.byteLength,
name: "00000000-0000-0000-0000-000000000000.tar.gz",
url: expect.stringContaining(
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.tar.gz"
),
extension: "tar.gz",
key: expect.stringContaining(
"app-id/00000000-0000-0000-0000-000000000000.tar.gz"
),
})
})
it("uploads file with non ascii filename to object store and returns signed URL", async () => {
const responseData = Buffer.from("teest file contnt")
const contentType = "text/plain"
const mockReadable = new Readable()
mockReadable.push(responseData)
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
headers: {
raw: () => ({
"content-type": [contentType],
"content-disposition": [
// eslint-disable-next-line no-useless-escape
`attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`,
],
}),
get: (header: any) => {
if (header === "content-type") return contentType
if (header === "content-disposition")
// eslint-disable-next-line no-useless-escape
return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`
},
},
body: mockReadable,
})
)
const query = {
path: "api",
}
const response = await config.integration.read(query)
expect(response.data).toEqual({
size: responseData.byteLength,
name: "00000000-0000-0000-0000-000000000000.pdf",
url: expect.stringContaining(
"/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.pdf"
),
extension: "pdf",
key: expect.stringContaining(
"app-id/00000000-0000-0000-0000-000000000000.pdf"
),
})
})
})
})

58
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -1,14 +1,27 @@
import { SqlClient } from "../utils"
import Sql from "../base/sql"
import {
FieldType,
Operation,
QueryJson,
TableSourceType,
Table,
FieldType,
TableSourceType,
} from "@budibase/types"
const TABLE_NAME = "test"
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {
id: {
name: "id",
type: FieldType.NUMBER,
},
},
name: TABLE_NAME,
primary: ["id"],
}
function endpoint(table: any, operation: any) {
return {
@ -25,6 +38,10 @@ function generateReadJson({
sort,
paginate,
}: any = {}): QueryJson {
const tableObj = { ...TABLE }
if (table) {
tableObj.name = table
}
return {
endpoint: endpoint(table || TABLE_NAME, "READ"),
resource: {
@ -34,14 +51,7 @@ function generateReadJson({
sort: sort || {},
paginate: paginate || {},
meta: {
table: {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: table || TABLE_NAME,
primary: ["id"],
} as any,
table: tableObj,
},
}
}
@ -49,6 +59,9 @@ function generateReadJson({
function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
return {
endpoint: endpoint(table, "CREATE"),
meta: {
table: TABLE,
},
body,
}
}
@ -58,7 +71,15 @@ function generateUpdateJson({
body = {},
filters = {},
meta = {},
}: {
table: string
body?: any
filters?: any
meta?: any
}): QueryJson {
if (!meta.table) {
meta.table = TABLE
}
return {
endpoint: endpoint(table, "UPDATE"),
filters,
@ -70,6 +91,9 @@ function generateUpdateJson({
function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
return {
endpoint: endpoint(table, "DELETE"),
meta: {
table: TABLE,
},
filters,
}
}
@ -102,6 +126,9 @@ function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}
@ -136,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}
@ -319,7 +349,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -338,7 +368,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -572,7 +602,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2000-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -591,7 +621,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2010-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})

22
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -4,12 +4,26 @@ import {
QueryJson,
SourceName,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
import sdk from "../../sdk"
// this doesn't exist strictly
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: "tableName",
primary: ["id"],
}
const AliasTables = sdk.rows.AliasTables
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
@ -103,7 +117,8 @@ describe("Captures of real examples", () => {
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
const filters = queryJson.filters
const notEqualsValue = Object.values(filters?.notEqual!)[0]
const rangeValue = Object.values(filters?.range!)[0]
const rangeValue: { high?: string | number; low?: string | number } =
Object.values(filters?.range!)[0]
const equalValue = Object.values(filters?.equal!)[0]
expect(query).toEqual({
@ -220,6 +235,9 @@ describe("Captures of real examples", () => {
resource: {
fields,
},
meta: {
table: TABLE,
},
}
}

2
packages/server/src/integrations/utils/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from "./utils"
export { SqlStatements } from "./sqlStatements"

80
packages/server/src/integrations/utils/sqlStatements.ts Normal file
View File

@ -0,0 +1,80 @@
import { FieldType, Table, FieldSchema } from "@budibase/types"
import { SqlClient } from "./utils"
import { Knex } from "knex"
export class SqlStatements {
client: string
table: Table
allOr: boolean | undefined
constructor(
client: string,
table: Table,
{ allOr }: { allOr?: boolean } = {}
) {
this.client = client
this.table = table
this.allOr = allOr
}
getField(key: string): FieldSchema | undefined {
const fieldName = key.split(".")[1]
return this.table.schema[fieldName]
}
between(
query: Knex.QueryBuilder,
key: string,
low: number | string,
high: number | string
) {
// Use a between operator if we have 2 valid range values
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[low, high]
)
} else {
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [low, high])
}
return query
}
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
// Use just a single greater than operator if we only have a low
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
low,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", low)
}
return query
}
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
const field = this.getField(key)
// Use just a single less than operator if we only have a high
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
high,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", high)
}
return query
}
}

251
packages/server/src/integrations/utils.ts → packages/server/src/integrations/utils/utils.ts
View File

@ -4,18 +4,45 @@ import {
Datasource,
FieldType,
TableSourceType,
FieldSchema,
} from "@budibase/types"
import { DocumentType, SEPARATOR } from "../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { helpers } from "@budibase/shared-core"
import env from "../environment"
import { context, objectStore } from "@budibase/backend-core"
import { v4 } from "uuid"
import { parseStringPromise as xmlParser } from "xml2js"
import { formatBytes } from "../../utilities"
import bl from "bl"
import env from "../../environment"
import { DocumentType, SEPARATOR } from "../../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
import { helpers, utils } from "@budibase/shared-core"
import { Knex } from "knex"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = {
type PrimitiveTypes =
| FieldType.STRING
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.DATETIME
| FieldType.JSON
| FieldType.BIGINT
| FieldType.OPTIONS
function isPrimitiveType(type: FieldType): type is PrimitiveTypes {
return [
FieldType.STRING,
FieldType.NUMBER,
FieldType.BOOLEAN,
FieldType.DATETIME,
FieldType.JSON,
FieldType.BIGINT,
FieldType.OPTIONS,
].includes(type)
}
const SQL_NUMBER_TYPE_MAP: Record<string, PrimitiveTypes> = {
integer: FieldType.NUMBER,
int: FieldType.NUMBER,
decimal: FieldType.NUMBER,
@ -35,7 +62,7 @@ const SQL_NUMBER_TYPE_MAP = {
smallmoney: FieldType.NUMBER,
}
const SQL_DATE_TYPE_MAP = {
const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
timestamp: FieldType.DATETIME,
time: FieldType.DATETIME,
datetime: FieldType.DATETIME,
@ -46,7 +73,7 @@ const SQL_DATE_TYPE_MAP = {
const SQL_DATE_ONLY_TYPES = ["date"]
const SQL_TIME_ONLY_TYPES = ["time"]
const SQL_STRING_TYPE_MAP = {
const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
varchar: FieldType.STRING,
char: FieldType.STRING,
nchar: FieldType.STRING,
@ -58,22 +85,22 @@ const SQL_STRING_TYPE_MAP = {
text: FieldType.STRING,
}
const SQL_BOOLEAN_TYPE_MAP = {
const SQL_BOOLEAN_TYPE_MAP: Record<string, PrimitiveTypes> = {
boolean: FieldType.BOOLEAN,
bit: FieldType.BOOLEAN,
tinyint: FieldType.BOOLEAN,
}
const SQL_OPTIONS_TYPE_MAP = {
const SQL_OPTIONS_TYPE_MAP: Record<string, PrimitiveTypes> = {
"user-defined": FieldType.OPTIONS,
}
const SQL_MISC_TYPE_MAP = {
const SQL_MISC_TYPE_MAP: Record<string, PrimitiveTypes> = {
json: FieldType.JSON,
bigint: FieldType.BIGINT,
}
const SQL_TYPE_MAP = {
const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {
...SQL_NUMBER_TYPE_MAP,
...SQL_DATE_TYPE_MAP,
...SQL_STRING_TYPE_MAP,
@ -239,14 +266,14 @@ export function generateColumnDefinition(config: {
constraints.inclusion = options
}
const schema: any = {
const schema: FieldSchema = {
type: foundType,
externalType,
autocolumn,
name,
constraints,
}
if (foundType === FieldType.DATETIME) {
if (schema.type === FieldType.DATETIME) {
schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)
schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType)
}
@ -274,59 +301,6 @@ export function isIsoDateString(str: string) {
return d.toISOString() === trimmedValue
}
/**
* This function will determine whether a column is a relationship and whether it
* is currently valid. The reason for the validity check is that tables can be deleted
* outside of Budibase control and if this is the case it will break Budibase relationships.
* The tableIds is a list passed down from the main finalise tables function, which is
* based on the tables that have just been fetched. This will only really be used on subsequent
* fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
* @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist.
*/
export function shouldCopyRelationship(
column: { type: string; tableId?: string },
tableIds: string[]
) {
return (
column.type === FieldType.LINK &&
column.tableId &&
tableIds.includes(column.tableId)
)
}
/**
* Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
* types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
* sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
* column has changed type in the external database then copying it over may not be possible.
* @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database.
*/
export function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
) {
const isFormula = column.type === FieldType.FORMULA
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
// column has been deleted, remove - formulas will never exist, always copy
if (!isFormula && column && !fetchedColumn) {
return false
}
const fetchedIsNumber =
!fetchedColumn || fetchedColumn.type === FieldType.NUMBER
return (
specialTypes.indexOf(column.type as FieldType) !== -1 ||
(fetchedIsNumber && column.type === FieldType.BOOLEAN)
)
}
/**
* Looks for columns which need to be copied over into the new table definitions, like relationships,
* options types and views.
@ -348,6 +322,9 @@ function copyExistingPropsOver(
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
if (entities[tableName]?.constrained) {
table.constrained = entities[tableName]?.constrained
}
table.views = entities[tableName].views
@ -356,12 +333,73 @@ function copyExistingPropsOver(
if (!Object.prototype.hasOwnProperty.call(existingTableSchema, key)) {
continue
}
const column = existingTableSchema[key]
if (
shouldCopyRelationship(column, tableIds) ||
shouldCopySpecialColumn(column, table.schema[key])
) {
table.schema[key] = existingTableSchema[key]
const existingColumnType = column?.type
const updatedColumnType = table.schema[key]?.type
const keepIfType = (...validTypes: PrimitiveTypes[]) => {
return (
isPrimitiveType(updatedColumnType) &&
table.schema[key] &&
validTypes.includes(updatedColumnType)
)
}
let shouldKeepSchema = false
switch (existingColumnType) {
case FieldType.FORMULA:
case FieldType.AUTO:
case FieldType.INTERNAL:
shouldKeepSchema = true
break
case FieldType.LINK:
shouldKeepSchema =
existingColumnType === FieldType.LINK &&
tableIds.includes(column.tableId)
break
case FieldType.STRING:
case FieldType.OPTIONS:
case FieldType.LONGFORM:
case FieldType.BARCODEQR:
shouldKeepSchema = keepIfType(FieldType.STRING)
break
case FieldType.NUMBER:
case FieldType.BOOLEAN:
shouldKeepSchema = keepIfType(FieldType.BOOLEAN, FieldType.NUMBER)
break
case FieldType.ARRAY:
case FieldType.ATTACHMENTS:
case FieldType.ATTACHMENT_SINGLE:
case FieldType.JSON:
case FieldType.BB_REFERENCE:
shouldKeepSchema = keepIfType(FieldType.JSON, FieldType.STRING)
break
case FieldType.DATETIME:
shouldKeepSchema = keepIfType(FieldType.DATETIME, FieldType.STRING)
break
case FieldType.BIGINT:
shouldKeepSchema = keepIfType(FieldType.BIGINT, FieldType.NUMBER)
break
default:
utils.unreachable(existingColumnType)
}
if (shouldKeepSchema) {
table.schema[key] = {
...existingTableSchema[key],
externalType:
existingTableSchema[key].externalType ||
table.schema[key]?.externalType,
}
}
}
}
@ -434,3 +472,74 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
export function isValidFilter(value: any) {
return value != null && value !== ""
}
export async function handleXml(response: any) {
let data,
rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
return { data, rawXml }
}
export async function handleFileResponse(
response: any,
filename: string,
startTime: number
) {
let presignedUrl,
size = 0
const fileExtension = filename.includes(".")
? filename.split(".").slice(1).join(".")
: ""
const processedFileName = `${v4()}.${fileExtension}`
const key = `${context.getProdAppId()}/${processedFileName}`
const bucket = objectStore.ObjectStoreBuckets.TEMP
const stream = response.body.pipe(bl((error, data) => data))
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
} else {
const chunks: Buffer[] = []
for await (const chunk of response.body) {
chunks.push(chunk)
size += chunk.length
}
}
await objectStore.streamUpload({
bucket,
filename: key,
stream,
ttl: 1,
type: response.headers["content-type"],
})
}
presignedUrl = await objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
}
}

1
packages/server/src/jsRunner/index.ts
View File

@ -35,7 +35,6 @@ export function init() {
// Because we can't pass functions into an Isolate, we remove them from
// the passed context and rely on the withHelpers() method to add them
// back in.
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { helpers, snippets, ...rest } = ctx
return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) {

3
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -348,8 +348,7 @@ const preSaveAction: Partial<Record<SourceName, any>> = {
* Make sure all datasource entities have a display name selected
*/
export function setDefaultDisplayColumns(datasource: Datasource) {
//
for (let entity of Object.values(datasource.entities || {})) {
for (const entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) {
continue
}

2
packages/server/src/sdk/app/rows/index.ts
View File

@ -3,6 +3,7 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
import AliasTables from "./sqlAlias"
export default {
...attachments,
@ -10,4 +11,5 @@ export default {
...search,
utils,
external,
AliasTables,
}

11
packages/server/src/sdk/app/rows/search.ts
View File

@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
export { isValidFilter } from "../../../integrations/utils"
@ -76,12 +78,15 @@ export async function search(
}
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (isExternalTable) {
return external.search(options)
return external.search(options, table)
} else if (env.SQS_SEARCH_ENABLE) {
return sqs.search(options)
return sqs.search(options, table)
} else {
return internal.search(options)
return internal.search(options, table)
}
}

13
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -8,6 +8,7 @@ import {
SearchFilters,
RowSearchParams,
SearchResponse,
Table,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import { handleRequest } from "../../../../api/controllers/row/external"
@ -18,13 +19,13 @@ import {
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { HTTPError, db } from "@budibase/backend-core"
import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
@ -68,8 +69,6 @@ export async function search(
}
try {
const table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
let rows = await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
@ -150,11 +149,15 @@ export async function exportRows(
}
const datasource = await sdk.datasources.get(datasourceId!)
const table = await sdk.tables.getTable(tableId)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query: requestQuery, sort, sortOrder })
let result = await search(
{ tableId, query: requestQuery, sort, sortOrder },
table
)
let rows: Row[] = []
let headers

Some files were not shown because too many files have changed in this diff Show More