diff --git a/.eslintrc.json b/.eslintrc.json index 525072dc6c..2c810eecc5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -42,7 +42,17 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "local-rules/no-budibase-imports": "error", + "local-rules/no-console-error": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-budibase-imports": "error" } }, @@ -59,7 +69,15 @@ }, "rules": { "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": "error", + "@typescript-eslint/no-unused-vars": [ + "error", + { + "varsIgnorePattern": "^_", + "argsIgnorePattern": "^_", + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true + } + ], "local-rules/no-test-com": "error", "local-rules/email-domain-example-com": "error", "no-console": "warn", @@ -89,7 +107,8 @@ { "varsIgnorePattern": "^_", "argsIgnorePattern": "^_", - "destructuredArrayIgnorePattern": "^_" + "destructuredArrayIgnorePattern": "^_", + "ignoreRestSiblings": true } ], "import/no-relative-packages": "error", diff --git a/.vscode/settings.json b/.vscode/settings.json index e22d5a8866..0723219a8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,8 @@ }, "[svelte]": { "editor.defaultFormatter": "svelte.svelte-vscode" + }, + "[handlebars]": { + "editor.formatOnSave": false } } diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index b380908dd1..ed7166ec5d 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -106,6 +106,8 @@ spec: value: {{ .Values.services.objectStore.globalBucketName | quote }} - name: BACKUPS_BUCKET_NAME value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: TEMP_BUCKET_NAME + value: {{ .Values.globals.tempBucketName | quote }} - name: PORT value: {{ .Values.services.apps.port | quote }} {{ if .Values.services.worker.publicApiRateLimitPerSecond }} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 51fa9ee4bb..3c6f94ae9e 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -107,6 +107,8 @@ spec: value: {{ .Values.services.objectStore.globalBucketName | quote }} - name: BACKUPS_BUCKET_NAME value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: TEMP_BUCKET_NAME + value: {{ .Values.globals.tempBucketName | quote }} - name: PORT value: {{ .Values.services.automationWorkers.port | quote }} {{ if .Values.services.worker.publicApiRateLimitPerSecond }} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index e37b2bc0e4..66a9bb6c14 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -106,6 +106,8 @@ spec: value: {{ .Values.services.objectStore.globalBucketName | quote }} - name: BACKUPS_BUCKET_NAME value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: TEMP_BUCKET_NAME + value: {{ .Values.globals.tempBucketName | quote }} - name: PORT value: {{ .Values.services.worker.port | quote }} - name: MULTI_TENANCY diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 9ace768625..27037cdaa8 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -121,6 +121,9 @@ globals: # to the old value for the duration of the rotation. jwtSecretFallback: "" + ## -- If using S3 the bucket name to be used for storing temporary files + tempBucketName: "" + smtp: # -- Whether to enable SMTP or not. enabled: false diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index a4866bc1f8..e88642c905 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -1,4 +1,25 @@ module.exports = { + "no-console-error": { + create: function(context) { + return { + CallExpression(node) { + if ( + node.callee.type === "MemberExpression" && + node.callee.object.name === "console" && + node.callee.property.name === "error" && + node.arguments.length === 1 && + node.arguments[0].name && + node.arguments[0].name.startsWith("err") + ) { + context.report({ + node, + message: 'Using console.error(err) on its own is not allowed. Either provide context to the error (console.error(msg, err)) or throw it.', + }) + } + }, + }; + }, + }, "no-budibase-imports": { create: function (context) { return { diff --git a/hosting/.env b/hosting/.env index 8a0756c0e3..173d409d04 100644 --- a/hosting/.env +++ b/hosting/.env @@ -17,6 +17,7 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_DB_SQS_PORT=4006 REDIS_PORT=6379 WATCHTOWER_PORT=6161 BUDIBASE_ENVIRONMENT=PRODUCTION @@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD= # A path that is watched for plugin bundles. Any bundles found are imported automatically/ PLUGINS_DIR= -ROLLING_LOG_MAX_SIZE= \ No newline at end of file +ROLLING_LOG_MAX_SIZE= diff --git a/lerna.json b/lerna.json index 9839b8b166..728cddc194 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.5", + "version": "2.23.11", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } diff --git a/package.json b/package.json index e520b7c2cf..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/packages/account-portal b/packages/account-portal index bd0e01d639..c167c331ff 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344 +Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964 diff --git a/packages/backend-core/src/auth/auth.ts b/packages/backend-core/src/auth/auth.ts index 87ac46cf1c..098e874863 100644 --- a/packages/backend-core/src/auth/auth.ts +++ b/packages/backend-core/src/auth/auth.ts @@ -64,7 +64,6 @@ async function refreshOIDCAccessToken( } strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp) } catch (err) { - console.error(err) throw new Error("Could not refresh OAuth Token") } @@ -99,7 +98,6 @@ async function refreshGoogleAccessToken( ssoSaveUserNoOp ) } catch (err: any) { - console.error(err) throw new Error( `Error constructing OIDC refresh strategy: message=${err.message}` ) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 4a2cfd34e2..d9dddd0097 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -8,19 +8,9 @@ import { SearchParams, WithRequired, } from "@budibase/types" +import { dataFilters } from "@budibase/shared-core" -const QUERY_START_REGEX = /\d[0-9]*:/g - -export function removeKeyNumbering(key: any): string { - if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) { - const parts = key.split(":") - // remove the number - parts.shift() - return parts.join(":") - } else { - return key - } -} +export const removeKeyNumbering = dataFilters.removeKeyNumbering /** * Class to build lucene query URLs. diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2da2a77d67..9ade81b9d7 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -29,6 +29,7 @@ const DefaultBucketName = { TEMPLATES: "templates", GLOBAL: "global", PLUGINS: "plugins", + TEMP: "tmp-file-attachments", } const selfHosted = !!parseInt(process.env.SELF_HOSTED || "") @@ -107,7 +108,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", - COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, @@ -146,6 +147,7 @@ const environment = { process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL, PLUGIN_BUCKET_NAME: process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS, + TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP, USE_COUCH: process.env.USE_COUCH || true, MOCK_REDIS: process.env.MOCK_REDIS, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index d357dbdbdc..69dba27c43 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -138,7 +138,6 @@ export default function ( } catch (err: any) { authenticated = false console.error(`Auth Error: ${err.message}`) - console.error(err) // remove the cookie as the user does not exist anymore clearCookie(ctx, Cookie.Auth) } @@ -187,7 +186,6 @@ export default function ( } } catch (err: any) { console.error(`Auth Error: ${err.message}`) - console.error(err) // invalid token, clear the cookie if (err?.name === "JsonWebTokenError") { clearCookie(ctx, Cookie.Auth) diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 2b8f7195ed..08f9f3214d 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -12,7 +12,7 @@ export async function errorHandling(ctx: any, next: any) { if (status >= 400 && status < 500) { console.warn(err) } else { - console.error(err) + console.error("Got 400 response code", err) } let error: APIError = { diff --git a/packages/backend-core/src/middleware/passport/sso/google.ts b/packages/backend-core/src/middleware/passport/sso/google.ts index 2a08ad7665..37a043cf0b 100644 --- a/packages/backend-core/src/middleware/passport/sso/google.ts +++ b/packages/backend-core/src/middleware/passport/sso/google.ts @@ -68,7 +68,6 @@ export async function strategyFactory( verify ) } catch (err: any) { - console.error(err) throw new Error(`Error constructing google authentication strategy: ${err}`) } } diff --git a/packages/backend-core/src/middleware/passport/sso/oidc.ts b/packages/backend-core/src/middleware/passport/sso/oidc.ts index 061e0507aa..35e6ee9fb0 100644 --- a/packages/backend-core/src/middleware/passport/sso/oidc.ts +++ b/packages/backend-core/src/middleware/passport/sso/oidc.ts @@ -103,7 +103,6 @@ export async function strategyFactory( strategy.name = "oidc" return strategy } catch (err: any) { - console.error(err) throw new Error(`Error constructing OIDC authentication strategy - ${err}`) } } @@ -142,7 +141,6 @@ export async function fetchStrategyConfig( callbackURL: callbackUrl, } } catch (err) { - console.error(err) throw new Error( `Error constructing OIDC authentication configuration - ${err}` ) diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts index 3f033b8cdb..fe6bc17386 100644 --- a/packages/backend-core/src/migrations/migrations.ts +++ b/packages/backend-core/src/migrations/migrations.ts @@ -26,7 +26,6 @@ export const getMigrationsDoc = async (db: any) => { if (err.status && err.status === 404) { return { _id: DocumentType.MIGRATIONS } } else { - console.error(err) throw err } } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 8d18fb97fd..aa5365c5c3 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -7,31 +7,41 @@ import tar from "tar-fs" import zlib from "zlib" import { promisify } from "util" import { join } from "path" -import fs, { ReadStream } from "fs" +import fs, { PathLike, ReadStream } from "fs" import env from "../environment" -import { budibaseTempDir } from "./utils" +import { bucketTTLConfig, budibaseTempDir } from "./utils" import { v4 } from "uuid" import { APP_PREFIX, APP_DEV_PREFIX } from "../db" +import fsp from "fs/promises" const streamPipeline = promisify(stream.pipeline) // use this as a temporary store of buckets that are being created const STATE = { bucketCreationPromises: {}, } +const signedFilePrefix = "/files/signed" type ListParams = { ContinuationToken?: string } -type UploadParams = { +type BaseUploadParams = { bucket: string filename: string - path: string type?: string | null - // can be undefined, we will remove it - metadata?: { - [key: string]: string | undefined - } + metadata?: { [key: string]: string | undefined } + body?: ReadableStream | Buffer + ttl?: number + addTTL?: boolean + extra?: any +} + +type UploadParams = BaseUploadParams & { + path?: string | PathLike +} + +type StreamUploadParams = BaseUploadParams & { + stream: ReadStream } const CONTENT_TYPE_MAP: any = { @@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = { js: "application/javascript", json: "application/json", gz: "application/gzip", + svg: "image/svg+xml", + form: "multipart/form-data", } const STRING_CONTENT_TYPES = [ @@ -105,7 +117,10 @@ export function ObjectStore( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export async function makeSureBucketExists(client: any, bucketName: string) { +export async function createBucketIfNotExists( + client: any, + bucketName: string +): Promise<{ created: boolean; exists: boolean }> { bucketName = sanitizeBucket(bucketName) try { await client @@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) { Bucket: bucketName, }) .promise() + return { created: false, exists: true } } catch (err: any) { const promises: any = STATE.bucketCreationPromises const doesntExist = err.statusCode === 404, noAccess = err.statusCode === 403 if (promises[bucketName]) { await promises[bucketName] + return { created: false, exists: true } } else if (doesntExist || noAccess) { if (doesntExist) { - // bucket doesn't exist create it promises[bucketName] = client .createBucket({ Bucket: bucketName, @@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) { .promise() await promises[bucketName] delete promises[bucketName] + return { created: true, exists: false } + } else { + throw new Error("Access denied to object store bucket." + err) } } else { throw new Error("Unable to write to object store bucket.") } } } - /** * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). @@ -146,12 +164,22 @@ export async function upload({ path, type, metadata, + body, + ttl, }: UploadParams) { const extension = filename.split(".").pop() - const fileBytes = fs.readFileSync(path) + + const fileBytes = path ? (await fsp.open(path)).createReadStream() : body const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) + + if (ttl && (bucketCreated.created || bucketCreated.exists)) { + let ttlConfig = bucketTTLConfig(bucketName, ttl) + if (objectStore.putBucketLifecycleConfiguration) { + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() + } + } let contentType = type if (!contentType) { @@ -174,6 +202,7 @@ export async function upload({ } config.Metadata = metadata } + return objectStore.upload(config).promise() } @@ -181,14 +210,24 @@ export async function upload({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export async function streamUpload( - bucketName: string, - filename: string, - stream: ReadStream | ReadableStream, - extra = {} -) { +export async function streamUpload({ + bucket: bucketName, + stream, + filename, + type, + extra, + ttl, +}: StreamUploadParams) { + const extension = filename.split(".").pop() const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + const bucketCreated = await createBucketIfNotExists(objectStore, bucketName) + + if (ttl && (bucketCreated.created || bucketCreated.exists)) { + let ttlConfig = bucketTTLConfig(bucketName, ttl) + if (objectStore.putBucketLifecycleConfiguration) { + await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise() + } + } // Set content type for certain known extensions if (filename?.endsWith(".js")) { @@ -203,10 +242,18 @@ export async function streamUpload( } } + let contentType = type + if (!contentType) { + contentType = extension + ? CONTENT_TYPE_MAP[extension.toLowerCase()] + : CONTENT_TYPE_MAP.txt + } + const params = { Bucket: sanitizeBucket(bucketName), Key: sanitizeKey(filename), Body: stream, + ContentType: contentType, ...extra, } return objectStore.upload(params).promise() @@ -286,7 +333,7 @@ export function getPresignedUrl( const signedUrl = new URL(url) const path = signedUrl.pathname const query = signedUrl.search - return `/files/signed${path}${query}` + return `${signedFilePrefix}${path}${query}` } } @@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) { */ export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + await createBucketIfNotExists(objectStore, bucketName) const params = { Bucket: bucketName, Key: sanitizeKey(filepath), @@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) { export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) - await makeSureBucketExists(objectStore, bucketName) + await createBucketIfNotExists(objectStore, bucketName) const params = { Bucket: bucketName, Delete: { @@ -412,7 +459,13 @@ export async function uploadDirectory( if (file.isDirectory()) { uploads.push(uploadDirectory(bucketName, local, path)) } else { - uploads.push(streamUpload(bucketName, path, fs.createReadStream(local))) + uploads.push( + streamUpload({ + bucket: bucketName, + filename: path, + stream: fs.createReadStream(local), + }) + ) } } await Promise.all(uploads) @@ -467,3 +520,23 @@ export async function getReadStream( } return client.getObject(params).createReadStream() } + +/* +Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract +the bucket and the path from it +*/ +export function extractBucketAndPath( + url: string +): { bucket: string; path: string } | null { + const baseUrl = url.split("?")[0] + + const regex = new RegExp(`^${signedFilePrefix}/(?[^/]+)/(?.+)$`) + const match = baseUrl.match(regex) + + if (match && match.groups) { + const { bucket, path } = match.groups + return { bucket, path } + } + + return null +} diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 4c3a84ba91..08b5238ff6 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -2,6 +2,7 @@ import { join } from "path" import { tmpdir } from "os" import fs from "fs" import env from "../environment" +import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3" /**************************************************** * NOTE: When adding a new bucket - name * @@ -15,6 +16,7 @@ export const ObjectStoreBuckets = { TEMPLATES: env.TEMPLATES_BUCKET_NAME, GLOBAL: env.GLOBAL_BUCKET_NAME, PLUGINS: env.PLUGIN_BUCKET_NAME, + TEMP: env.TEMP_BUCKET_NAME, } const bbTmp = join(tmpdir(), ".budibase") @@ -29,3 +31,27 @@ try { export function budibaseTempDir() { return bbTmp } + +export const bucketTTLConfig = ( + bucketName: string, + days: number +): PutBucketLifecycleConfigurationRequest => { + const lifecycleRule = { + ID: `${bucketName}-ExpireAfter${days}days`, + Prefix: "", + Status: "Enabled", + Expiration: { + Days: days, + }, + } + const lifecycleConfiguration = { + Rules: [lifecycleRule], + } + + const params = { + Bucket: bucketName, + LifecycleConfiguration: lifecycleConfiguration, + } + + return params +} diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 87e43b324d..333accc985 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -115,7 +115,6 @@ class InMemoryQueue implements Partial { * a JSON message as this is required by Bull. * @param repeat serves no purpose for the import queue. */ - // eslint-disable-next-line no-unused-vars async add(data: any, opts?: JobOptions) { const jobId = opts?.jobId?.toString() if (jobId && this._queuedJobIds.has(jobId)) { @@ -166,8 +165,7 @@ class InMemoryQueue implements Partial { return [] } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async removeJobs(pattern: string) { + async removeJobs(_pattern: string) { // no-op } diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 6165a68e57..f77c6385ba 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -50,6 +50,8 @@ type CreateAdminUserOpts = { hashPassword?: boolean requirePassword?: boolean skipPasswordValidation?: boolean + firstName?: string + lastName?: string } type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } @@ -517,6 +519,8 @@ export class UserDB { global: true, }, tenantId, + firstName: opts?.firstName, + lastName: opts?.lastName, } if (opts?.ssoId) { user.ssoId = opts.ssoId diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 48920a3771..7d62a6ef39 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,8 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchQuery, - SearchQueryOperators, + SearchFilters, + SearchFilterOperator, SearchUsersRequest, User, } from "@budibase/types" @@ -44,11 +44,11 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchQuery) { +export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ - { op: SearchQueryOperators.STRING, key: "email" }, - { op: SearchQueryOperators.EQUAL, key: "_id" }, - { op: SearchQueryOperators.ONE_OF, key: "_id" }, + { op: SearchFilterOperator.STRING, key: "email" }, + { op: SearchFilterOperator.EQUAL, key: "_id" }, + { op: SearchFilterOperator.ONE_OF, key: "_id" }, ] for (let [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { diff --git a/packages/backend-core/tests/core/utilities/index.ts b/packages/backend-core/tests/core/utilities/index.ts index 787d69be2c..b2f19a0286 100644 --- a/packages/backend-core/tests/core/utilities/index.ts +++ b/packages/backend-core/tests/core/utilities/index.ts @@ -4,3 +4,6 @@ export { generator } from "./structures" export * as testContainerUtils from "./testContainerUtils" export * as utils from "./utils" export * from "./jestUtils" +import * as minio from "./minio" + +export const objectStoreTestProviders = { minio } diff --git a/packages/backend-core/tests/core/utilities/minio.ts b/packages/backend-core/tests/core/utilities/minio.ts new file mode 100644 index 0000000000..cef33daa91 --- /dev/null +++ b/packages/backend-core/tests/core/utilities/minio.ts @@ -0,0 +1,34 @@ +import { GenericContainer, Wait, StartedTestContainer } from "testcontainers" +import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy" +import env from "../../../src/environment" + +let container: StartedTestContainer | undefined + +class ObjectStoreWaitStrategy extends AbstractWaitStrategy { + async waitUntilReady(container: any, boundPorts: any, startTime?: Date) { + const logs = Wait.forListeningPorts() + await logs.waitUntilReady(container, boundPorts, startTime) + } +} + +export async function start(): Promise { + container = await new GenericContainer("minio/minio") + .withExposedPorts(9000) + .withCommand(["server", "/data"]) + .withEnvironment({ + MINIO_ACCESS_KEY: "budibase", + MINIO_SECRET_KEY: "budibase", + }) + .withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000)) + .start() + + const port = container.getMappedPort(9000) + env._set("MINIO_URL", `http://0.0.0.0:${port}`) +} + +export async function stop() { + if (container) { + await container.stop() + container = undefined + } +} diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 0632993cf0..2d2022299c 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -14,6 +14,7 @@ notifications, Checkbox, DatePicker, + DrawerContent, } from "@budibase/bbui" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import { automationStore, selectedAutomation, tables } from "stores/builder" @@ -31,13 +32,14 @@ import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte" + import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import { BindingHelpers, BindingType } from "components/common/bindings/utils" import { bindingsToCompletions, hbAutocomplete, EditorModes, } from "components/common/CodeEditor" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" import { LuceneUtils, Utils } from "@budibase/frontend-core" import { getSchemaForDatasourcePlus, @@ -355,7 +357,8 @@ value.customType !== "queryParams" && value.customType !== "cron" && value.customType !== "triggerSchema" && - value.customType !== "automationFields" + value.customType !== "automationFields" && + value.type !== "attachment" ) } @@ -371,6 +374,15 @@ console.error(error) } }) + const handleAttachmentParams = keyValuObj => { + let params = {} + if (keyValuObj?.length) { + for (let param of keyValuObj) { + params[param.url] = param.filename + } + } + return params + }
@@ -436,21 +448,49 @@ value={inputData[key]} options={Object.keys(table?.schema || {})} /> + {:else if value.type === "attachment"} +
+
+ +
+
+ + onChange( + { + detail: e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + key + )} + object={handleAttachmentParams(inputData[key])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + /> +
+
{:else if value.customType === "filters"} Define filters - (tempFilters = e.detail)} - /> + + (tempFilters = e.detail)} + /> + {:else if value.customType === "password"} import { createEventDispatcher } from "svelte" import { ActionButton, Modal, ModalContent } from "@budibase/bbui" - import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" + import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" export let schema export let filters @@ -40,7 +40,7 @@ onConfirm={() => dispatch("change", tempValue)} >
- t.toLowerCase()) + return Object.entries(FIELDS) + .filter(([fieldType]) => + possibleTypes.includes(fieldType.toLowerCase()) + ) + .map(([_, fieldDefinition]) => fieldDefinition) } const isUsers = @@ -633,7 +627,7 @@ />
- {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} diff --git a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte index 496bee14ec..4a453ef049 100644 --- a/packages/builder/src/components/common/users/PasswordRepeatInput.svelte +++ b/packages/builder/src/components/common/users/PasswordRepeatInput.svelte @@ -9,7 +9,6 @@ "", requiredValidator ) - // eslint-disable-next-line no-unused-vars const [repeatPassword, _, repeatTouched] = createValidationStore( "", requiredValidator diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte new file mode 100644 index 0000000000..0ab67cbada --- /dev/null +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -0,0 +1,84 @@ + + + +
+ + { + const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id) + rawFilters[indexToUpdate] = { + ...rawFilters[indexToUpdate], + value: event.detail, + } + }} + /> + diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index 0f1f08d823..e481bb4381 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -1,8 +1,14 @@ -
- - - {#if !filters?.length} - Add your first filter expression. - {:else} - Results are filtered to only those which match all of the following - constraints. - {/if} - - {#if filters?.length} -
- {#each filters as filter} - onOperatorChange(filter, e.detail)} - placeholder={null} - /> - {#if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)} - - {:else if ["options", "array"].includes(filter.type)} - - {:else if filter.type === "boolean"} - - {:else if filter.type === "datetime"} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
- {/if} -
- -
- -
- - + +
+ Results are filtered to only those which match all of the following + constraints. +
+ diff --git a/packages/client/src/components/app/forms/AttachmentField.svelte b/packages/client/src/components/app/forms/AttachmentField.svelte index 644630810d..3489fd809c 100644 --- a/packages/client/src/components/app/forms/AttachmentField.svelte +++ b/packages/client/src/components/app/forms/AttachmentField.svelte @@ -58,17 +58,6 @@ } } - const deleteAttachments = async fileList => { - try { - return await API.deleteAttachments({ - keys: fileList, - tableId: formContext?.dataSource?.tableId, - }) - } catch (error) { - return [] - } - } - const handleChange = e => { const value = fieldApiMapper.set(e.detail) const changed = fieldApi.setValue(value) @@ -98,7 +87,6 @@ error={fieldState.error} on:change={handleChange} {processFiles} - {deleteAttachments} {handleFileTooLarge} {handleTooManyFiles} {maximum} diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 4ca88de8f2..3f97573d4a 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -11,6 +11,7 @@ "@budibase/types": "0.0.0", "dayjs": "^1.10.8", "lodash": "4.17.21", + "shortid": "2.2.15", "socket.io-client": "^4.6.1" } } diff --git a/packages/frontend-core/src/api/attachments.js b/packages/frontend-core/src/api/attachments.js index cff466c86f..72f280d99d 100644 --- a/packages/frontend-core/src/api/attachments.js +++ b/packages/frontend-core/src/api/attachments.js @@ -61,34 +61,6 @@ export const buildAttachmentEndpoints = API => { }) return { publicUrl } }, - - /** - * Deletes attachments from the bucket. - * @param keys the attachments to delete - * @param tableId the associated table ID - */ - deleteAttachments: async ({ keys, tableId }) => { - return await API.post({ - url: `/api/attachments/${tableId}/delete`, - body: { - keys, - }, - }) - }, - - /** - * Deletes attachments from the builder bucket. - * @param keys the attachments to delete - */ - deleteBuilderAttachments: async keys => { - return await API.post({ - url: `/api/attachments/delete`, - body: { - keys, - }, - }) - }, - /** * Download an attachment from a row given its column name. * @param datasourceId the ID of the datasource to download from diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte similarity index 50% rename from packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte rename to packages/frontend-core/src/components/FilterBuilder.svelte index 7f1ee8010d..074c2dbd9b 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -4,33 +4,36 @@ Button, Combobox, DatePicker, - DrawerContent, Icon, Input, - Label, Layout, - Multiselect, Select, + Label, + Multiselect, } from "@budibase/bbui" - import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" - import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" + import { FieldType, SearchFilterOperator } from "@budibase/types" import { generate } from "shortid" - import { Constants, LuceneUtils } from "@budibase/frontend-core" - import { getFields } from "helpers/searchFields" - import { FieldType } from "@budibase/types" - import { createEventDispatcher, onMount } from "svelte" + import { LuceneUtils, Constants } from "@budibase/frontend-core" + import { getContext } from "svelte" import FilterUsers from "./FilterUsers.svelte" + const { OperatorOptions } = Constants + export let schemaFields export let filters = [] - export let bindings = [] - export let panel = ClientBindingPanel - export let allowBindings = true export let datasource + export let behaviourFilters = false + export let allowBindings = false + export let filtersLabel = "Filters" + + $: matchAny = filters?.find(filter => filter.operator === "allOr") != null + $: onEmptyFilter = + filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" + + $: fieldFilters = filters.filter( + filter => filter.operator !== "allOr" && !filter.onEmptyFilter + ) - const dispatch = createEventDispatcher() - const { OperatorOptions } = Constants - const KeyedFieldRegex = /\d[0-9]*:/g const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, @@ -40,62 +43,18 @@ { value: "none", label: "Return no rows" }, ] - let rawFilters - let matchAny = false - let onEmptyFilter = "all" + const context = getContext("context") - $: parseFilters(filters) - $: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter)) - $: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true }) - $: fieldOptions = enrichedSchemaFields.map(field => field.name) || [] - $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] - - // Remove field key prefixes and determine which behaviours to use - const parseFilters = filters => { - matchAny = filters?.find(filter => filter.operator === "allOr") != null - onEmptyFilter = - filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" - rawFilters = (filters || []) - .filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter) - .map(filter => { - const { field } = filter - let newFilter = { ...filter } - delete newFilter.allOr - if (typeof field === "string" && field.match(KeyedFieldRegex) != null) { - const parts = field.split(":") - parts.shift() - newFilter.field = parts.join(":") - } - return newFilter - }) - } - - onMount(() => { - parseFilters(filters) - rawFilters.forEach(filter => { - filter.type = - schemaFields.find(field => field.name === filter.field)?.type || - filter.type - }) - }) - - // Add field key prefixes and a special metadata filter object to indicate - // how to handle filter behaviour - const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => { - let count = 1 - return rawFilters - .filter(filter => filter.field) - .map(filter => ({ - ...filter, - field: `${count++}:${filter.field}`, - })) - .concat(matchAny ? [{ operator: "allOr" }] : []) - .concat([{ onEmptyFilter }]) - } + $: fieldOptions = (schemaFields ?? []) + .filter(field => getValidOperatorsForType(field).length) + .map(field => ({ + label: field.displayName || field.name, + value: field.name, + })) const addFilter = () => { - rawFilters = [ - ...rawFilters, + filters = [ + ...(filters || []), { id: generate(), field: null, @@ -107,22 +66,62 @@ } const removeFilter = id => { - rawFilters = rawFilters.filter(field => field.id !== id) + filters = filters.filter(field => field.id !== id) + + // Clear all filters when no fields are specified + if (filters.length === 1 && filters[0].onEmptyFilter) { + filters = [] + } } const duplicateFilter = id => { - const existingFilter = rawFilters.find(filter => filter.id === id) + const existingFilter = filters.find(filter => filter.id === id) const duplicate = { ...existingFilter, id: generate() } - rawFilters = [...rawFilters, duplicate] + filters = [...filters, duplicate] + } + + const onFieldChange = filter => { + const previousType = filter.type + sanitizeTypes(filter) + sanitizeOperator(filter) + sanitizeValue(filter, previousType) + } + + const onOperatorChange = filter => { + sanitizeOperator(filter) + sanitizeValue(filter, filter.type) + } + + const onValueTypeChange = filter => { + sanitizeValue(filter) + } + + const getFieldOptions = field => { + const schema = schemaFields.find(x => x.name === field) + return schema?.constraints?.inclusion || [] } const getSchema = filter => { - return enrichedSchemaFields.find(field => field.name === filter.field) + return schemaFields.find(field => field.name === filter.field) } + const getValidOperatorsForType = filter => { + if (!filter?.field && !filter?.name) { + return [] + } + + return LuceneUtils.getValidOperatorsForType( + filter, + filter.field || filter.name, + datasource + ) + } + + $: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"] + const sanitizeTypes = filter => { // Update type based on field - const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field) + const fieldSchema = schemaFields.find(x => x.name === filter.field) filter.type = fieldSchema?.type filter.subtype = fieldSchema?.subtype @@ -154,88 +153,79 @@ // Ensure array values are properly set and cleared if (Array.isArray(filter.value)) { - if (filter.valueType !== "Value" || filter.type !== "array") { + if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) { filter.value = null } - } else if (filter.type === "array" && filter.valueType === "Value") { + } else if ( + filter.type === FieldType.ARRAY && + filter.valueType === "Value" + ) { filter.value = [] } else if ( previousType !== filter.type && (previousType === FieldType.BB_REFERENCE || filter.type === FieldType.BB_REFERENCE) ) { - filter.value = filter.type === "array" ? [] : null + filter.value = filter.type === FieldType.ARRAY ? [] : null } } - const onFieldChange = filter => { - const previousType = filter.type - sanitizeTypes(filter) - sanitizeOperator(filter) - sanitizeValue(filter, previousType) - } - - const onOperatorChange = filter => { - sanitizeOperator(filter) - sanitizeValue(filter, filter.type) - } - - const onValueTypeChange = filter => { - sanitizeValue(filter) - } - - const getFieldOptions = field => { - const schema = enrichedSchemaFields.find(x => x.name === field) - return schema?.constraints?.inclusion || [] - } - - const getValidOperatorsForType = filter => { - if (!filter?.field) { - return [] + function handleAllOr(option) { + filters = filters.filter(f => f.operator !== "allOr") + if (option === "or") { + filters.push({ operator: "allOr" }) } + } - return LuceneUtils.getValidOperatorsForType( - { type: filter.type, subtype: filter.subtype }, - filter.field, - datasource - ) + function handleOnEmptyFilter(value) { + filters = filters?.filter(filter => !filter.onEmptyFilter) + filters.push({ onEmptyFilter: value }) } - -
- - {#if !rawFilters?.length} - Add your first filter expression. - {:else} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => (onEmptyFilter = e.detail)} - placeholder={null} - /> +
+ + {#if fieldOptions?.length} + + {#if !fieldFilters?.length} + Add your first filter expression. + {:else} + + {#if behaviourFilters} +
+ opt.label} + getOptionValue={opt => opt.value} + on:change={e => handleOnEmptyFilter(e.detail)} + placeholder={null} + /> + {/if} +
{/if} -
+ {/if} + + {#if fieldFilters?.length}
-
- -
-
- {#each rawFilters as filter} + {#if filtersLabel} +
+ +
+ {/if} +
+ {#each fieldFilters as filter} onValueTypeChange(filter)} - placeholder={null} - /> - {#if filter.field && filter.valueType === "Binding"} - (filter.value = event.detail)} + {#if allowBindings} + - {:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")} + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)} - {:else if filter.type === "options"} + {:else if filter.type === FieldType.OPTIONS} - {:else if filter.type === "boolean"} + {:else if filter.type === FieldType.BOOLEAN} - {:else if filter.type === "datetime"} + {:else if filter.type === FieldType.DATETIME} {:else} - + {/if} - duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> +
+ duplicateFilter(filter.id)} + /> + removeFilter(filter.id)} + /> +
{/each}
{/if} -
+
- -
- + {:else} + None of the table column can be used for filtering. + {/if} + +
diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte similarity index 88% rename from packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte rename to packages/frontend-core/src/components/FilterUsers.svelte index 88383ba170..1712d7ebdf 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -1,9 +1,9 @@