MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into create-secret-key-once

This commit is contained in:
Sam Rose 2024-11-18 11:10:53 +00:00 committed by GitHub
parent 8c9092cc41 250303c8b7
commit 0a65d7245c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
57 changed files with 3313 additions and 3631 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hosting/proxy/nginx.prod.conf
View File

@ -50,19 +50,6 @@ http {
ignore_invalid_headers off;
proxy_buffering off;
set $csp_default "default-src 'self'";
set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
set $csp_manifest "manifest-src 'self'";
set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live";
set $csp_worker "worker-src blob:";
error_page 502 503 504 /error.html;
location = /error.html {
root /usr/share/nginx/html;
@ -73,7 +60,6 @@ http {
add_header X-Frame-Options SAMEORIGIN always;
add_header X-Content-Type-Options nosniff always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
# upstreams
@ -120,6 +106,12 @@ http {
location ~ ^/api/(system|admin|global)/ {
proxy_set_header Host $host;
# Enable buffering for potentially large OIDC configs
proxy_buffering on;
proxy_buffer_size 16k;
proxy_buffers 4 32k;
proxy_pass $worker;
}

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "3.2.0",
"version": "3.2.3",
"npmClient": "yarn",
"packages": [
"packages/*",

50
packages/backend-core/src/cache/tests/docWritethrough.spec.ts vendored
View File

@ -1,7 +1,12 @@
import tk from "timekeeper"
import _ from "lodash"
import { DBTestConfiguration, generator, structures } from "../../../tests"
import {
DBTestConfiguration,
generator,
structures,
utils,
} from "../../../tests"
import { getDB } from "../../db"
import {
@ -10,15 +15,14 @@ import {
init,
} from "../docWritethrough"
import InMemoryQueue from "../../queue/inMemoryQueue"
const initialTime = Date.now()
async function waitForQueueCompletion() {
const queue: InMemoryQueue = DocWritethroughProcessor.queue as never
await queue.waitForCompletion()
await utils.queue.processMessages(DocWritethroughProcessor.queue)
}
beforeAll(() => utils.queue.useRealQueues())
describe("docWritethrough", () => {
beforeAll(() => {
init()
@ -67,7 +71,7 @@ describe("docWritethrough", () => {
const patch3 = generatePatchObject(3)
await docWritethrough.patch(patch3)
expect(await db.get(documentId)).toEqual({
expect(await db.tryGet(documentId)).toEqual({
_id: documentId,
...patch1,
...patch2,
@ -92,7 +96,7 @@ describe("docWritethrough", () => {
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({
_id: documentId,
...patch1,
@ -117,7 +121,7 @@ describe("docWritethrough", () => {
await waitForQueueCompletion()
expect(date1).not.toEqual(date2)
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({
createdAt: date1.toISOString(),
updatedAt: date2.toISOString(),
@ -135,7 +139,7 @@ describe("docWritethrough", () => {
await docWritethrough.patch(patch2)
const keyToOverride = _.sample(Object.keys(patch1))!
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({
[keyToOverride]: patch1[keyToOverride],
})
@ -150,7 +154,7 @@ describe("docWritethrough", () => {
await docWritethrough.patch(patch3)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({
...patch1,
...patch2,
@ -180,14 +184,14 @@ describe("docWritethrough", () => {
await secondDocWritethrough.patch(doc2Patch2)
await waitForQueueCompletion()
expect(await db.get(docWritethrough.docId)).toEqual(
expect(await db.tryGet(docWritethrough.docId)).toEqual(
expect.objectContaining({
...doc1Patch,
...doc1Patch2,
})
)
expect(await db.get(secondDocWritethrough.docId)).toEqual(
expect(await db.tryGet(secondDocWritethrough.docId)).toEqual(
expect.objectContaining({
...doc2Patch,
...doc2Patch2,
@ -203,7 +207,7 @@ describe("docWritethrough", () => {
await docWritethrough.patch(initialPatch)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining(initialPatch)
)
@ -214,10 +218,10 @@ describe("docWritethrough", () => {
await docWritethrough.patch(extraPatch)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining(extraPatch)
)
expect(await db.get(documentId)).not.toEqual(
expect(await db.tryGet(documentId)).not.toEqual(
expect.objectContaining(initialPatch)
)
})
@ -242,7 +246,7 @@ describe("docWritethrough", () => {
expect(queueMessageSpy).toHaveBeenCalledTimes(5)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining(patches)
)
@ -250,7 +254,7 @@ describe("docWritethrough", () => {
expect(queueMessageSpy).toHaveBeenCalledTimes(45)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining(patches)
)
@ -258,20 +262,18 @@ describe("docWritethrough", () => {
expect(queueMessageSpy).toHaveBeenCalledTimes(55)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining(patches)
)
})
})
// This is not yet supported
// eslint-disable-next-line jest/no-disabled-tests
it.skip("patches will execute in order", async () => {
it("patches will execute in order", async () => {
let incrementalValue = 0
const keyToOverride = generator.word()
async function incrementalPatches(count: number) {
for (let i = 0; i < count; i++) {
await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
await docWritethrough.patch({ [keyToOverride]: ++incrementalValue })
}
}
@ -279,13 +281,13 @@ describe("docWritethrough", () => {
await incrementalPatches(5)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({ [keyToOverride]: 5 })
)
await incrementalPatches(40)
await waitForQueueCompletion()
expect(await db.get(documentId)).toEqual(
expect(await db.tryGet(documentId)).toEqual(
expect.objectContaining({ [keyToOverride]: 45 })
)
})

7
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -10,7 +10,6 @@ import {
DatabaseQueryOpts,
DBError,
Document,
FeatureFlag,
isDocument,
RowResponse,
RowValue,
@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
import { DDInstrumentedDatabase } from "../instrumentation"
import { checkSlashesInUrl } from "../../helpers"
import { sqlLog } from "../../sql/utils"
import { flags } from "../../features"
const DATABASE_NOT_FOUND = "Database does not exist."
@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
}
async destroy() {
if (
(await flags.isEnabled(FeatureFlag.SQS)) &&
(await this.exists(SQLITE_DESIGN_DOC_ID))
) {
if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
// delete the design document, then run the cleanup operation
const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
// remove all tables - save the definition then trigger a cleanup

4
packages/backend-core/src/environment.ts
View File

@ -230,6 +230,10 @@ const environment = {
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
MIN_VERSION_WITHOUT_POWER_ROLE:
process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0",
DISABLE_CONTENT_SECURITY_POLICY: process.env.DISABLE_CONTENT_SECURITY_POLICY,
// stopgap migration strategy until we can ensure backwards compat without unsafe-inline in CSP
DISABLE_CSP_UNSAFE_INLINE_SCRIPTS:
process.env.DISABLE_CSP_UNSAFE_INLINE_SCRIPTS,
}
export function setEnv(newEnvVars: Partial<typeof environment>): () => void {

2
packages/backend-core/src/features/features.ts
View File

@ -269,8 +269,6 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({
[FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
[FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
[FeatureFlag.SQS]: Flag.boolean(true),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
[FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
})

118
packages/backend-core/src/middleware/contentSecurityPolicy.ts Normal file
View File

@ -0,0 +1,118 @@
import crypto from "crypto"
import env from "../environment"
const CSP_DIRECTIVES = {
"default-src": ["'self'"],
"script-src": [
"'self'",
"'unsafe-eval'",
"https://*.budibase.net",
"https://cdn.budi.live",
"https://js.intercomcdn.com",
"https://widget.intercom.io",
"https://d2l5prqdbvm3op.cloudfront.net",
"https://us-assets.i.posthog.com",
],
"style-src": [
"'self'",
"'unsafe-inline'",
"https://cdn.jsdelivr.net",
"https://fonts.googleapis.com",
"https://rsms.me",
"https://maxcdn.bootstrapcdn.com",
],
"object-src": ["'none'"],
"base-uri": ["'self'"],
"connect-src": [
"'self'",
"https://*.budibase.app",
"https://*.budibaseqa.app",
"https://*.budibase.net",
"https://api-iam.intercom.io",
"https://api-ping.intercom.io",
"https://app.posthog.com",
"https://us.i.posthog.com",
"wss://nexus-websocket-a.intercom.io",
"wss://nexus-websocket-b.intercom.io",
"https://nexus-websocket-a.intercom.io",
"https://nexus-websocket-b.intercom.io",
"https://uploads.intercomcdn.com",
"https://uploads.intercomusercontent.com",
"https://*.amazonaws.com",
"https://*.s3.amazonaws.com",
"https://*.s3.us-east-2.amazonaws.com",
"https://*.s3.us-east-1.amazonaws.com",
"https://*.s3.us-west-1.amazonaws.com",
"https://*.s3.us-west-2.amazonaws.com",
"https://*.s3.af-south-1.amazonaws.com",
"https://*.s3.ap-east-1.amazonaws.com",
"https://*.s3.ap-south-1.amazonaws.com",
"https://*.s3.ap-northeast-2.amazonaws.com",
"https://*.s3.ap-southeast-1.amazonaws.com",
"https://*.s3.ap-southeast-2.amazonaws.com",
"https://*.s3.ap-northeast-1.amazonaws.com",
"https://*.s3.ca-central-1.amazonaws.com",
"https://*.s3.cn-north-1.amazonaws.com",
"https://*.s3.cn-northwest-1.amazonaws.com",
"https://*.s3.eu-central-1.amazonaws.com",
"https://*.s3.eu-west-1.amazonaws.com",
"https://*.s3.eu-west-2.amazonaws.com",
"https://*.s3.eu-south-1.amazonaws.com",
"https://*.s3.eu-west-3.amazonaws.com",
"https://*.s3.eu-north-1.amazonaws.com",
"https://*.s3.sa-east-1.amazonaws.com",
"https://*.s3.me-south-1.amazonaws.com",
"https://*.s3.us-gov-east-1.amazonaws.com",
"https://*.s3.us-gov-west-1.amazonaws.com",
"https://api.github.com",
],
"font-src": [
"'self'",
"data:",
"https://cdn.jsdelivr.net",
"https://fonts.gstatic.com",
"https://rsms.me",
"https://maxcdn.bootstrapcdn.com",
"https://js.intercomcdn.com",
"https://fonts.intercomcdn.com",
],
"frame-src": ["'self'", "https:"],
"img-src": ["http:", "https:", "data:", "blob:"],
"manifest-src": ["'self'"],
"media-src": [
"'self'",
"https://js.intercomcdn.com",
"https://cdn.budi.live",
],
"worker-src": ["blob:"],
}
export async function contentSecurityPolicy(ctx: any, next: any) {
try {
const nonce = crypto.randomBytes(16).toString("base64")
const directives = { ...CSP_DIRECTIVES }
directives["script-src"] = [
...CSP_DIRECTIVES["script-src"],
`'nonce-${nonce}'`,
]
if (!env.DISABLE_CSP_UNSAFE_INLINE_SCRIPTS) {
directives["script-src"].push("'unsafe-inline'")
}
ctx.state.nonce = nonce
const cspHeader = Object.entries(directives)
.map(([key, sources]) => `${key} ${sources.join(" ")}`)
.join("; ")
ctx.set("Content-Security-Policy", cspHeader)
await next()
} catch (err: any) {
console.error(
`Error occurred in Content-Security-Policy middleware: ${err}`
)
}
}
export default contentSecurityPolicy

1
packages/backend-core/src/middleware/index.ts
View File

@ -19,5 +19,6 @@ export { default as pino } from "../logging/pino/middleware"
export { default as correlation } from "../logging/correlation/middleware"
export { default as errorHandling } from "./errorHandling"
export { default as querystringToBody } from "./querystringToBody"
export { default as csp } from "./contentSecurityPolicy"
export * as joiValidator from "./joi-validator"
export { default as ip } from "./ip"

75
packages/backend-core/src/middleware/tests/contentSecurityPolicy.spec.ts Normal file
View File

@ -0,0 +1,75 @@
import crypto from "crypto"
import contentSecurityPolicy from "../contentSecurityPolicy"
jest.mock("crypto", () => ({
randomBytes: jest.fn(),
randomUUID: jest.fn(),
}))
describe("contentSecurityPolicy middleware", () => {
let ctx: any
let next: any
const mockNonce = "mocked/nonce"
beforeEach(() => {
ctx = {
state: {},
set: jest.fn(),
}
next = jest.fn()
// @ts-ignore
crypto.randomBytes.mockReturnValue(Buffer.from(mockNonce, "base64"))
})
afterEach(() => {
jest.clearAllMocks()
})
it("should generate a nonce and set it in the script-src directive", async () => {
await contentSecurityPolicy(ctx, next)
expect(ctx.state.nonce).toBe(mockNonce)
expect(ctx.set).toHaveBeenCalledWith(
"Content-Security-Policy",
expect.stringContaining(
`script-src 'self' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com 'nonce-${mockNonce}'`
)
)
expect(next).toHaveBeenCalled()
})
it("should include all CSP directives in the header", async () => {
await contentSecurityPolicy(ctx, next)
const cspHeader = ctx.set.mock.calls[0][1]
expect(cspHeader).toContain("default-src 'self'")
expect(cspHeader).toContain("script-src 'self' 'unsafe-eval'")
expect(cspHeader).toContain("style-src 'self' 'unsafe-inline'")
expect(cspHeader).toContain("object-src 'none'")
expect(cspHeader).toContain("base-uri 'self'")
expect(cspHeader).toContain("connect-src 'self'")
expect(cspHeader).toContain("font-src 'self'")
expect(cspHeader).toContain("frame-src 'self'")
expect(cspHeader).toContain("img-src http: https: data: blob:")
expect(cspHeader).toContain("manifest-src 'self'")
expect(cspHeader).toContain("media-src 'self'")
expect(cspHeader).toContain("worker-src blob:")
})
it("should handle errors and log an error message", async () => {
const consoleSpy = jest.spyOn(console, "error").mockImplementation()
const error = new Error("Test error")
// @ts-ignore
crypto.randomBytes.mockImplementation(() => {
throw error
})
await contentSecurityPolicy(ctx, next)
expect(consoleSpy).toHaveBeenCalledWith(
`Error occurred in Content-Security-Policy middleware: ${error}`
)
expect(next).not.toHaveBeenCalled()
consoleSpy.mockRestore()
})
})

12
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -1,5 +1,5 @@
import events from "events"
import { newid, timeout } from "../utils"
import { newid } from "../utils"
import { Queue, QueueOptions, JobOptions } from "./queue"
interface JobMessage {
@ -184,16 +184,6 @@ class InMemoryQueue implements Partial<Queue> {
// do nothing
return this as any
}
async waitForCompletion() {
do {
await timeout(50)
} while (this.hasRunningJobs())
}
hasRunningJobs() {
return this._addCount > this._runCount
}
}
export default InMemoryQueue

13
packages/backend-core/src/queue/queue.ts
View File

@ -15,7 +15,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
// cleanup the queue every 60 seconds
const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
let QUEUES: BullQueue.Queue[] = []
let cleanupInterval: NodeJS.Timeout
async function cleanup() {
@ -45,11 +45,18 @@ export function createQueue<T>(
if (opts.jobOptions) {
queueConfig.defaultJobOptions = opts.jobOptions
}
let queue: any
let queue: BullQueue.Queue<T>
if (!env.isTest()) {
queue = new BullQueue(jobQueue, queueConfig)
} else if (
process.env.BULL_TEST_REDIS_PORT &&
!isNaN(+process.env.BULL_TEST_REDIS_PORT)
) {
queue = new BullQueue(jobQueue, {
redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
})
} else {
queue = new InMemoryQueue(jobQueue, queueConfig)
queue = new InMemoryQueue(jobQueue, queueConfig) as any
}
addListeners(queue, jobQueue, opts?.removeStalledCb)
QUEUES.push(queue)

1
packages/backend-core/tests/core/utilities/index.ts
View File

@ -4,3 +4,4 @@ export { generator } from "./structures"
export * as testContainerUtils from "./testContainerUtils"
export * as utils from "./utils"
export * from "./jestUtils"
export * as queue from "./queue"

9
packages/backend-core/tests/core/utilities/queue.ts Normal file
View File

@ -0,0 +1,9 @@
import { Queue } from "bull"
export async function processMessages(queue: Queue) {
do {
await queue.whenCurrentJobsFinished()
} while (await queue.count())
await queue.whenCurrentJobsFinished()
}

57
packages/backend-core/tests/core/utilities/testContainerUtils.ts
View File

@ -1,4 +1,6 @@
import { execSync } from "child_process"
import { cloneDeep } from "lodash"
import { GenericContainer, StartedTestContainer } from "testcontainers"
const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
@ -106,3 +108,58 @@ export function setupEnv(...envs: any[]) {
}
}
}
export async function startContainer(container: GenericContainer) {
const imageName = (container as any).imageName.string as string
let key: string = imageName
if (imageName.includes("@sha256")) {
key = imageName.split("@")[0]
}
key = key.replace(/\//g, "-").replace(/:/g, "-")
container = container
.withReuse()
.withLabels({ "com.budibase": "true" })
.withName(`${key}_testcontainer`)
let startedContainer: StartedTestContainer | undefined = undefined
let lastError = undefined
for (let i = 0; i < 10; i++) {
try {
// container.start() is not an idempotent operation, calling `start`
// modifies the internal state of a GenericContainer instance such that
// the hash it uses to determine reuse changes. We need to clone the
// container before calling start to ensure that we're using the same
// reuse hash every time.
const containerCopy = cloneDeep(container)
startedContainer = await containerCopy.start()
lastError = undefined
break
} catch (e: any) {
lastError = e
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
if (!startedContainer) {
if (lastError) {
throw lastError
}
throw new Error(`failed to start container: ${imageName}`)
}
const info = getContainerById(startedContainer.getId())
if (!info) {
throw new Error("Container not found")
}
// Some Docker runtimes, when you expose a port, will bind it to both
// 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
// addresses are not shared, and testcontainers will sometimes give you back
// the ipv6 port. There's no way to know that this has happened, and if you
// try to then connect to `localhost:port` you may attempt to bind to the v4
// address which could be unbound or even an entirely different container. For
// that reason, we don't use testcontainers' `getExposedPort` function,
// preferring instead our own method that guaranteed v4 ports.
return getExposedV4Ports(info)
}

1
packages/backend-core/tests/core/utilities/utils/index.ts
View File

@ -1 +1,2 @@
export * as time from "./time"
export * as queue from "./queue"

27
packages/backend-core/tests/core/utilities/utils/queue.ts Normal file
View File

@ -0,0 +1,27 @@
import { Queue } from "bull"
import { GenericContainer, Wait } from "testcontainers"
import { startContainer } from "../testContainerUtils"
export async function useRealQueues() {
const ports = await startContainer(
new GenericContainer("redis")
.withExposedPorts(6379)
.withWaitStrategy(
Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000)
)
)
const port = ports.find(x => x.container === 6379)?.host
if (!port) {
throw new Error("Redis port not found")
}
process.env.BULL_TEST_REDIS_PORT = port.toString()
}
export async function processMessages(queue: Queue) {
do {
await queue.whenCurrentJobsFinished()
} while (await queue.count())
await queue.whenCurrentJobsFinished()
}

39
packages/bbui/src/Form/Core/Dropzone.svelte
View File

@ -8,6 +8,7 @@
import Link from "../../Link/Link.svelte"
import Tag from "../../Tags/Tag.svelte"
import Tags from "../../Tags/Tags.svelte"
import ProgressCircle from "../../ProgressCircle/ProgressCircle.svelte"
const BYTES_IN_KB = 1000
const BYTES_IN_MB = 1000000
@ -39,12 +40,14 @@
"jfif",
"webp",
]
const fieldId = id || uuid()
let selectedImageIdx = 0
let fileDragged = false
let selectedUrl
let fileInput
let loading = false
$: selectedImage = value?.[selectedImageIdx] ?? null
$: fileCount = value?.length ?? 0
$: isImage =
@ -86,10 +89,15 @@
}
if (processFiles) {
const processedFiles = await processFiles(fileList)
const newValue = [...value, ...processedFiles]
dispatch("change", newValue)
selectedImageIdx = newValue.length - 1
loading = true
try {
const processedFiles = await processFiles(fileList)
const newValue = [...value, ...processedFiles]
dispatch("change", newValue)
selectedImageIdx = newValue.length - 1
} finally {
loading = false
}
} else {
dispatch("change", fileList)
}
@ -227,7 +235,7 @@
{#if showDropzone}
<div
class="spectrum-Dropzone"
class:disabled
class:disabled={disabled || loading}
role="region"
tabindex="0"
on:dragover={handleDragOver}
@ -241,7 +249,7 @@
id={fieldId}
{disabled}
type="file"
multiple
multiple={maximum !== 1}
accept={extensions}
bind:this={fileInput}
on:change={handleFile}
@ -339,6 +347,12 @@
{/if}
{/if}
</div>
{#if loading}
<div class="loading">
<ProgressCircle size="M" />
</div>
{/if}
</div>
{/if}
</div>
@ -464,6 +478,7 @@
.spectrum-Dropzone {
height: 220px;
position: relative;
}
.compact .spectrum-Dropzone {
height: 40px;
@ -488,4 +503,14 @@
.tag {
margin-top: 8px;
}
.loading {
position: absolute;
display: grid;
place-items: center;
height: 100%;
width: 100%;
top: 0;
left: 0;
}
</style>

7
packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
View File

@ -11,7 +11,6 @@
export let disabledPermissions = []
export let columns
export let fromRelationshipField
export let canSetRelationshipSchemas
const { datasource, dispatch } = getContext("grid")
@ -129,6 +128,8 @@
}
})
$: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK)
async function toggleColumn(column, permission) {
const visible = permission !== FieldPermissions.HIDDEN
const readonly = permission === FieldPermissions.READONLY
@ -184,7 +185,7 @@
value={columnToPermissionOptions(column)}
options={column.options}
/>
{#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
{#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
<div class="relationship-columns">
<ActionButton
on:click={e => {
@ -203,7 +204,7 @@
</div>
</div>
{#if canSetRelationshipSchemas}
{#if hasLinkColumns}
<Popover
on:close={() => (relationshipFieldName = null)}
open={relationshipFieldName}

8
packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
View File

@ -10,8 +10,6 @@
import { getContext } from "svelte"
import { ActionButton } from "@budibase/bbui"
import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
import { isEnabled } from "helpers/featureFlags"
import { FeatureFlag } from "@budibase/types"
import DetailPopover from "components/common/DetailPopover.svelte"
const { tableColumns, datasource } = getContext("grid")
@ -46,9 +44,5 @@
{text}
</ActionButton>
</svelte:fragment>
<ColumnsSettingContent
columns={$tableColumns}
canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
{permissions}
/>
<ColumnsSettingContent columns={$tableColumns} {permissions} />
</DetailPopover>

2
packages/frontend-core/src/components/grid/layout/NewColumnButton.svelte
View File

@ -53,6 +53,7 @@
on:close={close}
maxHeight={null}
resizable
minWidth={360}
>
<div class="content">
<slot />
@ -80,7 +81,6 @@
}
.content {
width: 300px;
padding: 20px;
display: flex;
flex-direction: column;

1
packages/frontend-core/src/fetch/NestedProviderFetch.js
View File

@ -5,6 +5,7 @@ export default class NestedProviderFetch extends DataFetch {
// Nested providers should already have exposed their own schema
return {
schema: datasource?.value?.schema,
primaryDisplay: datasource?.value?.primaryDisplay,
}
}

2
packages/pro

@ -1 +1 @@
Subproject commit 04bee88597edb1edb88ed299d0597b587f0362ec
Subproject commit a56696a4af5667617746600fc75fe6a01744b692

6
packages/server/src/api/controllers/application.ts
View File

@ -153,7 +153,11 @@ async function createInstance(appId: string, template: AppTemplate) {
await createAllSearchIndex()
if (template && template.useTemplate) {
await sdk.backups.importApp(appId, db, template)
const opts = {
importObjStoreContents: true,
updateAttachmentColumns: !template.key, // preserve attachments when using Budibase templates
}
await sdk.backups.importApp(appId, db, template, opts)
} else {
// create the users table
await db.put(USERS_TABLE_SCHEMA)

3
packages/server/src/api/controllers/static/index.ts
View File

@ -209,6 +209,7 @@ export const serveApp = async function (ctx: UserCtx) {
? objectStore.getGlobalFileUrl("settings", "logoUrl")
: "",
appMigrating: needMigrations,
nonce: ctx.state.nonce,
})
const appHbs = loadHandlebarsFile(appHbsPath)
ctx.body = await processString(appHbs, {
@ -217,6 +218,7 @@ export const serveApp = async function (ctx: UserCtx) {
css: `:root{${themeVariables}} ${css.code}`,
appId,
embedded: bbHeaderEmbed,
nonce: ctx.state.nonce,
})
} else {
// just return the app info for jest to assert on
@ -258,6 +260,7 @@ export const serveBuilderPreview = async function (ctx: Ctx) {
const previewHbs = loadHandlebarsFile(join(previewLoc, "preview.hbs"))
ctx.body = await processString(previewHbs, {
clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
nonce: ctx.state.nonce,
})
} else {
// just return the app info for jest to assert on

8
packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
View File

@ -16,6 +16,8 @@
export let hideDevTools
export let sideNav
export let hideFooter
export let nonce
</script>
<svelte:head>
@ -118,11 +120,11 @@
<p />
{/if}
</div>
<script type="application/javascript">
<script type="application/javascript" {nonce}>
window.INIT_TIME = Date.now()
</script>
{#if appMigrating}
<script type="application/javascript">
<script type="application/javascript" {nonce}>
window.MIGRATING_APP = true
</script>
{/if}
@ -135,7 +137,7 @@
<script type="application/javascript" src={plugin.jsUrl}></script>
{/each}
{/if}
<script type="application/javascript">
<script type="application/javascript" {nonce}>
if (window.loadBudibase) {
window.loadBudibase()
} else {

4
packages/server/src/api/controllers/static/templates/app.hbs
View File

@ -1,5 +1,5 @@
<html>
<script>
<script nonce="{{ nonce }}">
document.fonts.ready.then(() => {
window.parent.postMessage({ type: "docLoaded" });
})
@ -9,7 +9,7 @@
<style>{{{css}}}</style>
</head>
<script>
<script nonce="{{ nonce }}">
window["##BUDIBASE_APP_ID##"] = "{{appId}}"
window["##BUDIBASE_APP_EMBEDDED##"] = "{{embedded}}"
</script>

2
packages/server/src/api/controllers/static/templates/preview.hbs
View File

@ -31,7 +31,7 @@
}
</style>
<script src='{{ clientLibPath }}'></script>
<script>
<script nonce="{{ nonce }}">
function receiveMessage(event) {
if (!event.data) {
return

13
packages/server/src/api/controllers/table/utils.ts
View File

@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils"
import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { context, events, features, HTTPError } from "@budibase/backend-core"
import { context, events, HTTPError } from "@budibase/backend-core"
import {
AutoFieldSubType,
Database,
Datasource,
FeatureFlag,
FieldSchema,
FieldType,
NumberFieldMetadata,
@ -336,9 +335,8 @@ class TableSaveFunctions {
importRows: this.importRows,
userId: this.userId,
})
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.addTable(table)
}
await sdk.tables.sqs.addTable(table)
return table
}
@ -530,9 +528,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
if (rows) {
await AttachmentCleanup.tableDelete(table, rows)
}
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.removeTable(table)
}
await sdk.tables.sqs.removeTable(table)
}
const _TableSaveFunctions = TableSaveFunctions

17
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -16,7 +16,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import { events, utils, context, features } from "@budibase/backend-core"
import { events, utils, context } from "@budibase/backend-core"
import env from "../../../environment"
import { type App, BuiltinPermissionID } from "@budibase/types"
import tk from "timekeeper"
@ -355,21 +355,6 @@ describe("/applications", () => {
expect(events.app.deleted).toHaveBeenCalledTimes(1)
expect(events.app.unpublished).toHaveBeenCalledTimes(1)
})
it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => {
const prodAppId = app.appId.replace("_dev", "")
nock("http://localhost:10000")
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})
describe("POST /api/applications/:appId/duplicate", () => {

831
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -9,27 +9,18 @@ import {
import tk from "timekeeper"
import emitter from "../../../../src/events"
import { outputProcessing } from "../../../utilities/rowProcessor"
import {
context,
InternalTable,
tenancy,
features,
utils,
} from "@budibase/backend-core"
import { context, InternalTable, tenancy, utils } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
AIOperationEnum,
AttachmentFieldMetadata,
AutoFieldSubType,
Datasource,
DateFieldMetadata,
DeleteRow,
FieldSchema,
FieldType,
BBReferenceFieldSubType,
FormulaType,
INTERNAL_TABLE_SOURCE_ID,
NumberFieldMetadata,
QuotaUsageType,
RelationshipType,
Row,
@ -90,8 +81,7 @@ async function waitForEvent(
}
describe.each([
["lucene", undefined],
["sqs", undefined],
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
@ -99,8 +89,6 @@ describe.each([
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/rows (%s)", (providerType, dsProvider) => {
const isInternal = dsProvider === undefined
const isLucene = providerType === "lucene"
const isSqs = providerType === "sqs"
const isMSSQL = providerType === DatabaseName.SQL_SERVER
const isOracle = providerType === DatabaseName.ORACLE
const config = setup.getConfig()
@ -108,15 +96,9 @@ describe.each([
let table: Table
let datasource: Datasource | undefined
let client: Knex | undefined
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
await config.init()
if (dsProvider) {
const rawDatasource = await dsProvider
@ -129,9 +111,6 @@ describe.each([
afterAll(async () => {
setup.afterAll()
if (envCleanup) {
envCleanup()
}
})
function saveTableRequest(
@ -381,185 +360,6 @@ describe.each([
expect(ids).toEqual(expect.arrayContaining(sequence))
})
isLucene &&
it("row values are coerced", async () => {
const str: FieldSchema = {
type: FieldType.STRING,
name: "str",
constraints: { type: "string", presence: false },
}
const singleAttachment: FieldSchema = {
type: FieldType.ATTACHMENT_SINGLE,
name: "single attachment",
constraints: { presence: false },
}
const attachmentList: AttachmentFieldMetadata = {
type: FieldType.ATTACHMENTS,
name: "attachments",
constraints: { type: "array", presence: false },
}
const signature: FieldSchema = {
type: FieldType.SIGNATURE_SINGLE,
name: "signature",
constraints: { presence: false },
}
const bool: FieldSchema = {
type: FieldType.BOOLEAN,
name: "boolean",
constraints: { type: "boolean", presence: false },
}
const number: NumberFieldMetadata = {
type: FieldType.NUMBER,
name: "str",
constraints: { type: "number", presence: false },
}
const datetime: DateFieldMetadata = {
type: FieldType.DATETIME,
name: "datetime",
constraints: {
type: "string",
presence: false,
datetime: { earliest: "", latest: "" },
},
}
const arrayField: FieldSchema = {
type: FieldType.ARRAY,
constraints: {
type: JsonFieldSubType.ARRAY,
presence: false,
inclusion: ["One", "Two", "Three"],
},
name: "Sample Tags",
sortable: false,
}
const optsField: FieldSchema = {
name: "Sample Opts",
type: FieldType.OPTIONS,
constraints: {
type: "string",
presence: false,
inclusion: ["Alpha", "Beta", "Gamma"],
},
}
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: str,
stringUndefined: str,
stringNull: str,
stringString: str,
numberEmptyString: number,
numberNull: number,
numberUndefined: number,
numberString: number,
numberNumber: number,
datetimeEmptyString: datetime,
datetimeNull: datetime,
datetimeUndefined: datetime,
datetimeString: datetime,
datetimeDate: datetime,
boolNull: bool,
boolEmpty: bool,
boolUndefined: bool,
boolString: bool,
boolBool: bool,
singleAttachmentNull: singleAttachment,
singleAttachmentUndefined: singleAttachment,
attachmentListNull: attachmentList,
attachmentListUndefined: attachmentList,
attachmentListEmpty: attachmentList,
attachmentListEmptyArrayStr: attachmentList,
signatureNull: signature,
signatureUndefined: signature,
arrayFieldEmptyArrayStr: arrayField,
arrayFieldArrayStrKnown: arrayField,
arrayFieldNull: arrayField,
arrayFieldUndefined: arrayField,
optsFieldEmptyStr: optsField,
optsFieldUndefined: optsField,
optsFieldNull: optsField,
optsFieldStrKnown: optsField,
},
})
)
const datetimeStr = "1984-04-20T00:00:00.000Z"
const row = await config.api.row.save(table._id!, {
name: "Test Row",
stringUndefined: undefined,
stringNull: null,
stringString: "i am a string",
numberEmptyString: "",
numberNull: null,
numberUndefined: undefined,
numberString: "123",
numberNumber: 123,
datetimeEmptyString: "",
datetimeNull: null,
datetimeUndefined: undefined,
datetimeString: datetimeStr,
datetimeDate: new Date(datetimeStr),
boolNull: null,
boolEmpty: "",
boolUndefined: undefined,
boolString: "true",
boolBool: true,
tableId: table._id,
singleAttachmentNull: null,
singleAttachmentUndefined: undefined,
attachmentListNull: null,
attachmentListUndefined: undefined,
attachmentListEmpty: "",
attachmentListEmptyArrayStr: "[]",
signatureNull: null,
signatureUndefined: undefined,
arrayFieldEmptyArrayStr: "[]",
arrayFieldUndefined: undefined,
arrayFieldNull: null,
arrayFieldArrayStrKnown: "['One']",
optsFieldEmptyStr: "",
optsFieldUndefined: undefined,
optsFieldNull: null,
optsFieldStrKnown: "Alpha",
})
expect(row.stringUndefined).toBe(undefined)
expect(row.stringNull).toBe(null)
expect(row.stringString).toBe("i am a string")
expect(row.numberEmptyString).toBe(null)
expect(row.numberNull).toBe(null)
expect(row.numberUndefined).toBe(undefined)
expect(row.numberString).toBe(123)
expect(row.numberNumber).toBe(123)
expect(row.datetimeEmptyString).toBe(null)
expect(row.datetimeNull).toBe(null)
expect(row.datetimeUndefined).toBe(undefined)
expect(row.datetimeString).toBe(new Date(datetimeStr).toISOString())
expect(row.datetimeDate).toBe(new Date(datetimeStr).toISOString())
expect(row.boolNull).toBe(null)
expect(row.boolEmpty).toBe(null)
expect(row.boolUndefined).toBe(undefined)
expect(row.boolString).toBe(true)
expect(row.boolBool).toBe(true)
expect(row.singleAttachmentNull).toEqual(null)
expect(row.singleAttachmentUndefined).toBe(undefined)
expect(row.attachmentListNull).toEqual([])
expect(row.attachmentListUndefined).toBe(undefined)
expect(row.attachmentListEmpty).toEqual([])
expect(row.attachmentListEmptyArrayStr).toEqual([])
expect(row.signatureNull).toEqual(null)
expect(row.signatureUndefined).toBe(undefined)
expect(row.arrayFieldEmptyArrayStr).toEqual([])
expect(row.arrayFieldNull).toEqual([])
expect(row.arrayFieldUndefined).toEqual(undefined)
expect(row.optsFieldEmptyStr).toEqual(null)
expect(row.optsFieldUndefined).toEqual(undefined)
expect(row.optsFieldNull).toEqual(null)
expect(row.arrayFieldArrayStrKnown).toEqual(["One"])
expect(row.optsFieldStrKnown).toEqual("Alpha")
})
isInternal &&
it("doesn't allow creating in user table", async () => {
const response = await config.api.row.save(
@ -1023,104 +823,103 @@ describe.each([
})
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can create rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
},
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: [
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
],
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
})
it("can create rows with no relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
})
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
it("can create rows with only one relationships field", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can create rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: [
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
],
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
})
it("can create rows with no relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
})
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
it("can create rows with only one relationships field", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
})
})
describe("get", () => {
@ -1224,133 +1023,132 @@ describe.each([
expect(rows).toHaveLength(1)
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can edit rows with both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
},
})
row = await config.api.row.save(table._id!, {
...row,
related1: [relatedRows[0]._id!, relatedRows[1]._id!],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: expect.arrayContaining([
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
]),
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
})
it("can drop existing relationship", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
it("can drop both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
})
)
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can edit rows with both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [relatedRows[0]._id!, relatedRows[1]._id!],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: expect.arrayContaining([
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
]),
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
})
it("can drop existing relationship", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
it("can drop both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
})
)
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
})
})
describe("patch", () => {
@ -1628,72 +1426,71 @@ describe.each([
expect(res.length).toEqual(2)
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can delete rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
it("can delete rows with empty relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [],
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
},
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can delete rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
it("can delete rows with empty relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [],
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
})
})
describe("validate", () => {
@ -3061,13 +2858,7 @@ describe.each([
let auxData: Row[] = []
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
flagCleanup = features.testutils.setFeatureFlags("*", {
ENRICHED_RELATIONSHIPS: true,
})
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -3214,10 +3005,6 @@ describe.each([
viewId = view.id
})
afterAll(() => {
flagCleanup?.()
})
const testScenarios: [string, (row: Row) => Promise<Row> | Row][] = [
["get row", (row: Row) => config.api.row.get(viewId, row._id!)],
[
@ -3290,68 +3077,6 @@ describe.each([
}
)
it.each(testScenarios)(
"does not enrich relationships when not enabled (via %s)",
async (__, retrieveDelegate) => {
await features.testutils.withFeatureFlags(
"*",
{
ENRICHED_RELATIONSHIPS: false,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)
const row = await config.api.row.save(viewId, {
title: generator.word(),
relWithNoSchema: [otherRows[0]],
relWithEmptySchema: [otherRows[1]],
relWithFullSchema: [otherRows[2]],
relWithHalfSchema: [otherRows[3]],
relWithIllegalSchema: [otherRows[4]],
})
const retrieved = await retrieveDelegate(row)
expect(retrieved).toEqual(
expect.objectContaining({
title: row.title,
relWithNoSchema: [
{
_id: otherRows[0]._id,
primaryDisplay: otherRows[0].name,
},
],
relWithEmptySchema: [
{
_id: otherRows[1]._id,
primaryDisplay: otherRows[1].name,
},
],
relWithFullSchema: [
{
_id: otherRows[2]._id,
primaryDisplay: otherRows[2].name,
},
],
relWithHalfSchema: [
{
_id: otherRows[3]._id,
primaryDisplay: otherRows[3].name,
},
],
relWithIllegalSchema: [
{
_id: otherRows[4]._id,
primaryDisplay: otherRows[4].name,
},
],
})
)
}
)
}
)
it.each([
[
"from table fetch",
@ -3422,7 +3147,7 @@ describe.each([
)
})
isSqs &&
isInternal &&
describe("AI fields", () => {
let table: Table

925
packages/server/src/api/routes/tests/search.spec.ts
View File

File diff suppressed because it is too large Load Diff

65
packages/server/src/api/routes/tests/templates.spec.ts
View File

@ -2,7 +2,6 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -82,48 +81,36 @@ describe("/templates", () => {
})
describe("create app from template", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
it("should be able to create an app from a template", async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
})
expect(rows).toHaveLength(3)
})
}
)
}
)
expect(rows).toHaveLength(3)
})
})
})
})

4100
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

File diff suppressed because it is too large Load Diff

89
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -1,10 +1,6 @@
import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
} from "@budibase/backend-core"
import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
import {
LinkDocument,
DocumentType,
@ -70,24 +66,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
}
async function sqsDisabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {
beforeAll(async () => {
await sqsDisabled(async () => {
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
beforeEach(async () => {
@ -101,43 +87,32 @@ describe("SQS migration", () => {
it("test migration runs as expected against an older DB", async () => {
const db = dbCore.getDB(config.appId!)
// confirm nothing exists initially
await sqsDisabled(async () => {
let error: any | undefined
try {
await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err: any) {
error = err
}
expect(error).toBeDefined()
expect(error.status).toBe(404)
// remove sqlite design doc to simulate it comes from an older installation
const doc = await db.get(SQLITE_DESIGN_DOC_ID)
await db.remove({ _id: doc._id, _rev: doc._rev })
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
await sqsEnabled(async () => {
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(
generateJunctionTableID(tableId1, tableId2)
)
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2))
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
})

11
packages/server/src/db/linkedRows/index.ts
View File

@ -14,11 +14,10 @@ import {
coreOutputProcessing,
processFormulas,
} from "../../utilities/rowProcessor"
import { context, features } from "@budibase/backend-core"
import { context } from "@budibase/backend-core"
import {
ContextUser,
EventType,
FeatureFlag,
FieldType,
LinkDocumentValue,
Row,
@ -251,19 +250,13 @@ export async function squashLinks<T = Row[] | Row>(
source: Table | ViewV2,
enriched: T
): Promise<T> {
const allowRelationshipSchemas = await features.flags.isEnabled(
FeatureFlag.ENRICHED_RELATIONSHIPS
)
let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) {
return enriched
}
if (allowRelationshipSchemas) {
viewSchema = source.schema || {}
}
viewSchema = source.schema || {}
}
let table: Table

59
packages/server/src/integrations/tests/utils/index.ts
View File

@ -6,12 +6,12 @@ import * as mysql from "./mysql"
import * as mssql from "./mssql"
import * as mariadb from "./mariadb"
import * as oracle from "./oracle"
import { GenericContainer, StartedTestContainer } from "testcontainers"
import { testContainerUtils } from "@budibase/backend-core/tests"
import cloneDeep from "lodash/cloneDeep"
export type DatasourceProvider = () => Promise<Datasource>
export const { startContainer } = testContainerUtils
export enum DatabaseName {
POSTGRES = "postgres",
MONGODB = "mongodb",
@ -71,58 +71,3 @@ export async function knexClient(ds: Datasource) {
}
}
}
export async function startContainer(container: GenericContainer) {
const imageName = (container as any).imageName.string as string
let key: string = imageName
if (imageName.includes("@sha256")) {
key = imageName.split("@")[0]
}
key = key.replaceAll("/", "-").replaceAll(":", "-")
container = container
.withReuse()
.withLabels({ "com.budibase": "true" })
.withName(`${key}_testcontainer`)
let startedContainer: StartedTestContainer | undefined = undefined
let lastError = undefined
for (let i = 0; i < 10; i++) {
try {
// container.start() is not an idempotent operation, calling `start`
// modifies the internal state of a GenericContainer instance such that
// the hash it uses to determine reuse changes. We need to clone the
// container before calling start to ensure that we're using the same
// reuse hash every time.
const containerCopy = cloneDeep(container)
startedContainer = await containerCopy.start()
lastError = undefined
break
} catch (e: any) {
lastError = e
await new Promise(resolve => setTimeout(resolve, 1000))
}
}
if (!startedContainer) {
if (lastError) {
throw lastError
}
throw new Error(`failed to start container: ${imageName}`)
}
const info = testContainerUtils.getContainerById(startedContainer.getId())
if (!info) {
throw new Error("Container not found")
}
// Some Docker runtimes, when you expose a port, will bind it to both
// 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
// addresses are not shared, and testcontainers will sometimes give you back
// the ipv6 port. There's no way to know that this has happened, and if you
// try to then connect to `localhost:port` you may attempt to bind to the v4
// address which could be unbound or even an entirely different container. For
// that reason, we don't use testcontainers' `getExposedPort` function,
// preferring instead our own method that guaranteed v4 ports.
return testContainerUtils.getExposedV4Ports(info)
}

11
packages/server/src/koa.ts
View File

@ -6,7 +6,13 @@ import * as api from "./api"
import * as automations from "./automations"
import { Thread } from "./threads"
import * as redis from "./utilities/redis"
import { events, logging, middleware, timers } from "@budibase/backend-core"
import {
events,
logging,
middleware,
timers,
env as coreEnv,
} from "@budibase/backend-core"
import destroyable from "server-destroy"
import { userAgent } from "koa-useragent"
@ -37,6 +43,9 @@ export default function createKoaApp() {
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
if (!coreEnv.DISABLE_CONTENT_SECURITY_POLICY) {
app.use(middleware.csp)
}
app.use(userAgent)
const server = http.createServer(app.callback())

1
packages/server/src/sdk/app/applications/import.ts
View File

@ -123,6 +123,7 @@ export async function updateWithExport(
// don't need obj store, the existing app already has everything we need
await backups.importApp(devId, tempDb, template, {
importObjStoreContents: false,
updateAttachmentColumns: true,
})
const newMetadata = await getNewAppMetadata(tempDb, appDb)
// get the documents to copy

9
packages/server/src/sdk/app/backups/imports.ts
View File

@ -170,7 +170,10 @@ export async function importApp(
appId: string,
db: Database,
template: TemplateType,
opts: { importObjStoreContents: boolean } = { importObjStoreContents: true }
opts: {
importObjStoreContents: boolean
updateAttachmentColumns: boolean
} = { importObjStoreContents: true, updateAttachmentColumns: true }
) {
let prodAppId = dbCore.getProdAppID(appId)
let dbStream: any
@ -219,7 +222,9 @@ export async function importApp(
if (!ok) {
throw "Error loading database dump from template."
}
await updateAttachmentColumns(prodAppId, db)
if (opts.updateAttachmentColumns) {
await updateAttachmentColumns(prodAppId, db)
}
await updateAutomations(prodAppId, db)
// clear up afterward
if (tmpPath) {

55
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,11 +1,8 @@
import {
EmptyFilterOption,
FeatureFlag,
LegacyFilter,
LogicalOperator,
Row,
RowSearchParams,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
@ -19,7 +16,6 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { checkFilters, searchInputMapping } from "./search/utils"
import { db, features } from "@budibase/backend-core"
import tracer from "dd-trace"
import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
import { enrichSearchContext } from "../../../api/controllers/row/utils"
@ -104,44 +100,14 @@ export async function search(
}
viewQuery = checkFilters(table, viewQuery)
const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
const supportsLogicalOperators =
isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
const { filters } = dataFilters.splitFiltersArray(queryFilters)
// Extract existing fields
const existingFields = filters.map(filter =>
db.removeKeyNumbering(filter.field)
)
// Carry over filters for unused fields
Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
viewQuery[operator]![field] = options.query[operator]![field]
}
})
})
options.query = viewQuery
} else {
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
}
@ -170,12 +136,9 @@ export async function search(
if (isExternalTable) {
span?.addTags({ searchType: "external" })
result = await external.search(options, source)
} else if (await features.flags.isEnabled(FeatureFlag.SQS)) {
} else {
span?.addTags({ searchType: "sqs" })
result = await internal.sqs.search(options, source)
} else {
span?.addTags({ searchType: "lucene" })
result = await internal.lucene.search(options, source)
}
span.addTags({

1
packages/server/src/sdk/app/rows/search/internal/index.ts
View File

@ -1,3 +1,2 @@
export * as sqs from "./sqs"
export * as lucene from "./lucene"
export * from "./internal"

79
packages/server/src/sdk/app/rows/search/internal/lucene.ts
View File

@ -1,79 +0,0 @@
import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import { fullSearch, paginatedSearch } from "../utils"
import { InternalTables } from "../../../../../db/utils"
import {
Row,
RowSearchParams,
SearchResponse,
SortType,
Table,
User,
ViewV2,
} from "@budibase/types"
import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
import { outputProcessing } from "../../../../../utilities/rowProcessor"
import pick from "lodash/pick"
import sdk from "../../../../"
export async function search(
options: RowSearchParams,
source: Table | ViewV2
): Promise<SearchResponse<Row>> {
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
const { paginate, query } = options
const params: RowSearchParams = {
tableId: options.tableId,
viewId: options.viewId,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
query: {},
}
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
params.sortType =
sortField.type === "number" ? SortType.NUMBER : SortType.STRING
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (table._id === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, allowedFields))
response.rows = await outputProcessing(source, response.rows, {
squash: true,
})
}
return response
}

78
packages/server/src/sdk/app/rows/search/tests/search.spec.ts
View File

@ -10,7 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -21,30 +21,20 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures"
// (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead.
describe.each([
["lucene", undefined],
["sqs", undefined],
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("search sdk (%s)", (name, dsProvider) => {
const isSqs = name === "sqs"
const isLucene = name === "lucene"
const isInternal = isLucene || isSqs
const isInternal = name === "internal"
const config = new TestConfiguration()
let envCleanup: (() => void) | undefined
let datasource: Datasource | undefined
let table: Table
beforeAll(async () => {
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
@ -105,9 +95,6 @@ describe.each([
afterAll(async () => {
config.end()
if (envCleanup) {
envCleanup()
}
})
it("querying by fields will always return data attribute columns", async () => {
@ -211,36 +198,35 @@ describe.each([
})
})
!isLucene &&
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
{ equal: { id: 7 } },
],
},
},
{ equal: { id: 7 } },
],
},
fields: queryFields,
})
expect(rows).toHaveLength(expectedRows)
},
fields: queryFields,
})
}
)
expect(rows).toHaveLength(expectedRows)
})
}
)
})

8
packages/server/src/sdk/app/tables/getters.ts
View File

@ -1,4 +1,4 @@
import { context, features } from "@budibase/backend-core"
import { context } from "@budibase/backend-core"
import { getTableParams } from "../../../db/utils"
import {
breakExternalTableId,
@ -12,7 +12,6 @@ import {
TableResponse,
TableSourceType,
TableViewsResponse,
FeatureFlag,
} from "@budibase/types"
import datasources from "../datasources"
import sdk from "../../../sdk"
@ -49,10 +48,7 @@ export async function processTable(table: Table): Promise<Table> {
type: "table",
sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
}
const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
if (sqsEnabled) {
processed.sql = true
sql: true,
}
return processed
}

1
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -237,6 +237,7 @@ export default class TestConfiguration {
if (!this) {
return
}
if (this.server) {
this.server.close()
} else {

62
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils"
import {
cache,
context,
features,
HTTPError,
objectStore,
utils,
@ -19,7 +18,6 @@ import {
Table,
User,
ViewV2,
FeatureFlag,
} from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
@ -423,45 +421,43 @@ export async function coreOutputProcessing(
// remove null properties to match internal API
const isExternal = isExternalTableID(table._id!)
if (isExternal || (await features.flags.isEnabled(FeatureFlag.SQS))) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
} else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
for (const link of row[key] || []) {
for (const linkKey of Object.keys(link)) {
if (link[linkKey] === null) {
delete link[linkKey]
}
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
} else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
for (const link of row[key] || []) {
for (const linkKey of Object.keys(link)) {
if (link[linkKey] === null) {
delete link[linkKey]
}
}
}
}
}
}
if (sdk.views.isView(source)) {
// We ensure calculation fields are returned as numbers. During the
// testing of this feature it was discovered that the COUNT operation
// returns a string for MySQL, MariaDB, and Postgres. But given that all
// calculation fields (except ones operating on BIGINTs) should be
// numbers, we blanket make sure of that here.
for (const [name, field] of Object.entries(
helpers.views.calculationFields(source)
)) {
if ("field" in field) {
const targetSchema = table.schema[field.field]
// We don't convert BIGINT fields to floats because we could lose
// precision.
if (targetSchema.type === FieldType.BIGINT) {
continue
}
if (sdk.views.isView(source)) {
// We ensure calculation fields are returned as numbers. During the
// testing of this feature it was discovered that the COUNT operation
// returns a string for MySQL, MariaDB, and Postgres. But given that all
// calculation fields (except ones operating on BIGINTs) should be
// numbers, we blanket make sure of that here.
for (const [name, field] of Object.entries(
helpers.views.calculationFields(source)
)) {
if ("field" in field) {
const targetSchema = table.schema[field.field]
// We don't convert BIGINT fields to floats because we could lose
// precision.
if (targetSchema.type === FieldType.BIGINT) {
continue
}
}
for (const row of rows) {
if (typeof row[name] === "string") {
row[name] = parseFloat(row[name])
}
for (const row of rows) {
if (typeof row[name] === "string") {
row[name] = parseFloat(row[name])
}
}
}

8
packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts
View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,6 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +32,6 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupFlags()
})
const processOutputBBReferenceMock =

7
packages/shared-core/src/filters.ts
View File

@ -527,7 +527,12 @@ export function search<T extends Record<string, any>>(
): SearchResponse<T> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
result = sort(
result,
query.sort,
query.sortOrder || SortOrder.ASCENDING,
query.sortType
)
}
const totalRows = result.length
if (query.limit) {

2
packages/shared-core/src/helpers/cron.ts
View File

@ -48,7 +48,7 @@ export function validate(
cronExpression: string
): { valid: false; err: string[] } | { valid: true } {
const result = cronValidate(cronExpression, {
preset: "npm-node-cron",
preset: "npm-cron-schedule",
override: {
useSeconds: false,
},

1
packages/types/src/sdk/db.ts
View File

@ -12,7 +12,6 @@ import type PouchDB from "pouchdb-find"
export enum SearchIndex {
ROWS = "rows",
AUDIT = "audit",
USER = "user",
}

3
packages/types/src/sdk/featureFlag.ts
View File

@ -2,10 +2,9 @@ export enum FeatureFlag {
PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE",
PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
SQS = "SQS",
AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
DEFAULT_VALUES = "DEFAULT_VALUES",
ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",
BUDIBASE_AI = "BUDIBASE_AI",
}

2
packages/types/src/sdk/koa.ts
View File

@ -48,6 +48,7 @@ export interface Ctx<RequestBody = any, ResponseBody = any> extends Context {
request: BBRequest<RequestBody>
body: ResponseBody
userAgent: UserAgentContext["userAgent"]
state: { nonce?: string }
}
/**
@ -56,6 +57,7 @@ export interface Ctx<RequestBody = any, ResponseBody = any> extends Context {
export interface UserCtx<RequestBody = any, ResponseBody = any>
extends Ctx<RequestBody, ResponseBody> {
user: ContextUser
state: { nonce?: string }
roleId?: string
eventEmitter?: ContextEmitter
loginMethod?: LoginMethod

9
packages/worker/src/api/controllers/system/environment.ts
View File

@ -1,6 +1,6 @@
import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
import { Ctx, MaintenanceType } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
import { helpers } from "@budibase/shared-core"
@ -35,10 +35,7 @@ async function isSqsAvailable() {
}
async function isSqsMissing() {
return (
(await features.flags.isEnabled(FeatureFlag.SQS)) &&
!(await isSqsAvailable())
)
return !(await isSqsAvailable())
}
export const fetch = async (ctx: Ctx) => {

9
packages/worker/src/api/routes/global/tests/auditLogs.spec.ts
View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, features } from "@budibase/backend-core"
import { context, events } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -12,19 +12,14 @@ const BASE_IDENTITY = {
const USER_AUDIT_LOG_COUNT = 3
const APP_ID = "app_1"
describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
describe("/api/global/auditlogs (%s)", () => {
const config = new TestConfiguration()
let envCleanup: (() => void) | undefined
beforeAll(async () => {
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})
afterAll(async () => {
envCleanup?.()
await config.afterAll()
})

3
packages/worker/src/index.ts
View File

@ -56,6 +56,9 @@ app.use(koaSession(app))
app.use(middleware.correlation)
app.use(middleware.pino)
app.use(middleware.ip)
if (!coreEnv.DISABLE_CONTENT_SECURITY_POLICY) {
app.use(middleware.csp)
}
app.use(userAgent)
// authentication

2
packages/worker/src/tests/TestConfiguration.ts
View File

@ -12,7 +12,7 @@ dbConfig.init()
import env from "../environment"
import * as controllers from "./controllers"
const supertest = require("supertest")
import supertest from "supertest"
import { Config } from "../constants"
import {

2
yarn.lock
View File

@ -23244,4 +23244,4 @@ zip-stream@^6.0.1:
dependencies:
archiver-utils "^5.0.0"
compress-commons "^6.0.2"
readable-stream "^4.0.0"
readable-stream "^4.0.0"