Merge remote-tracking branch 'origin/master' into feature/filter-bindings

This commit is contained in:
Dean 2024-05-09 11:05:58 +01:00
commit 4c6629d15f
36 changed files with 362 additions and 211 deletions

View File

@ -170,7 +170,8 @@ jobs:
docker pull mongo:7.0-jammy &
docker pull mariadb:lts &
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.2.1-sql &
docker pull budibase/couchdb:v3.2.1-sqs &
docker pull minio/minio &
docker pull redis &
wait $(jobs -p)

View File

@ -2,7 +2,7 @@
apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "budibase.fullname" . }}-apps
name: {{ include "budibase.fullname" . }}-automation-worker
labels:
{{- include "budibase.labels" . | nindent 4 }}
spec:

View File

@ -46,7 +46,7 @@ export default async function setup() {
await killContainers(containers)
try {
let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
const couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
.withExposedPorts(5984, 4984)
.withEnvironment({
COUCHDB_PASSWORD: "budibase",
@ -69,7 +69,20 @@ export default async function setup() {
).withStartupTimeout(20000)
)
await couchdb.start()
const minio = new GenericContainer("minio/minio")
.withExposedPorts(9000)
.withCommand(["server", "/data"])
.withEnvironment({
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
})
.withLabels({ "com.budibase": "true" })
.withReuse()
.withWaitStrategy(
Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000)
)
await Promise.all([couchdb.start(), minio.start()])
} finally {
lockfile.unlockSync(lockPath)
}

View File

@ -1,5 +1,5 @@
{
"version": "2.24.1",
"version": "2.24.2",
"npmClient": "yarn",
"packages": [
"packages/*",

View File

@ -12,6 +12,10 @@ import { dataFilters } from "@budibase/shared-core"
export const removeKeyNumbering = dataFilters.removeKeyNumbering
function isEmpty(value: any) {
return value == null || value === ""
}
/**
* Class to build lucene query URLs.
* Optionally takes a base lucene query object.
@ -282,15 +286,14 @@ export class QueryBuilder<T> {
}
const equal = (key: string, value: any) => {
// 0 evaluates to false, which means we would return all rows if we don't check it
if (!value && value !== 0) {
if (isEmpty(value)) {
return null
}
return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
}
const contains = (key: string, value: any, mode = "AND") => {
if (!value || (Array.isArray(value) && value.length === 0)) {
if (isEmpty(value)) {
return null
}
if (!Array.isArray(value)) {
@ -306,7 +309,7 @@ export class QueryBuilder<T> {
}
const fuzzy = (key: string, value: any) => {
if (!value) {
if (isEmpty(value)) {
return null
}
value = builder.preprocess(value, {
@ -328,7 +331,7 @@ export class QueryBuilder<T> {
}
const oneOf = (key: string, value: any) => {
if (!value) {
if (isEmpty(value)) {
return `*:*`
}
if (!Array.isArray(value)) {
@ -386,7 +389,7 @@ export class QueryBuilder<T> {
// Construct the actual lucene search query string from JSON structure
if (this.#query.string) {
build(this.#query.string, (key: string, value: any) => {
if (!value) {
if (isEmpty(value)) {
return null
}
value = builder.preprocess(value, {
@ -399,7 +402,7 @@ export class QueryBuilder<T> {
}
if (this.#query.range) {
build(this.#query.range, (key: string, value: any) => {
if (!value) {
if (isEmpty(value)) {
return null
}
if (value.low == null || value.low === "") {
@ -421,7 +424,7 @@ export class QueryBuilder<T> {
}
if (this.#query.notEqual) {
build(this.#query.notEqual, (key: string, value: any) => {
if (!value) {
if (isEmpty(value)) {
return null
}
if (typeof value === "boolean") {

View File

@ -83,7 +83,7 @@ export function ObjectStore(
bucket: string,
opts: { presigning: boolean } = { presigning: false }
) {
const config: any = {
const config: AWS.S3.ClientConfiguration = {
s3ForcePathStyle: true,
signatureVersion: "v4",
apiVersion: "2006-03-01",

View File

@ -4,6 +4,3 @@ export { generator } from "./structures"
export * as testContainerUtils from "./testContainerUtils"
export * as utils from "./utils"
export * from "./jestUtils"
import * as minio from "./minio"
export const objectStoreTestProviders = { minio }

View File

@ -1,34 +0,0 @@
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
import env from "../../../src/environment"
let container: StartedTestContainer | undefined
class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
const logs = Wait.forListeningPorts()
await logs.waitUntilReady(container, boundPorts, startTime)
}
}
export async function start(): Promise<void> {
container = await new GenericContainer("minio/minio")
.withExposedPorts(9000)
.withCommand(["server", "/data"])
.withEnvironment({
MINIO_ACCESS_KEY: "budibase",
MINIO_SECRET_KEY: "budibase",
})
.withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
.start()
const port = container.getMappedPort(9000)
env._set("MINIO_URL", `http://0.0.0.0:${port}`)
}
export async function stop() {
if (container) {
await container.stop()
container = undefined
}
}

View File

@ -86,10 +86,18 @@ export function setupEnv(...envs: any[]) {
throw new Error("CouchDB SQL port not found")
}
const minio = getContainerByImage("minio/minio")
const minioPort = getExposedV4Port(minio, 9000)
if (!minioPort) {
throw new Error("Minio port not found")
}
const configs = [
{ key: "COUCH_DB_PORT", value: `${couchPort}` },
{ key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
{ key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
{ key: "MINIO_URL", value: `http://127.0.0.1:${minioPort}` },
]
for (const config of configs.filter(x => !!x.value)) {

View File

@ -8,6 +8,8 @@
export let size = "S"
export let extraButtonText
export let extraButtonAction
export let extraLinkText
export let extraLinkAction
export let showCloseButton = true
let show = true
@ -28,8 +30,13 @@
<use xlink:href="#spectrum-icon-18-{icon}" />
</svg>
<div class="spectrum-Toast-body">
<div class="spectrum-Toast-content">
<div class="spectrum-Toast-content row-content">
<slot />
{#if extraLinkText}
<button class="link" on:click={extraLinkAction}>
<u>{extraLinkText}</u>
</button>
{/if}
</div>
{#if extraButtonText && extraButtonAction}
<button
@ -73,4 +80,23 @@
.spectrum-Button {
border: 1px solid rgba(255, 255, 255, 0.2);
}
.row-content {
display: flex;
}
.link {
background: none;
border: none;
margin: 0;
margin-left: 0.5em;
padding: 0;
cursor: pointer;
color: white;
font-weight: 600;
}
u {
font-weight: 600;
}
</style>

View File

@ -0,0 +1,43 @@
<script>
import "@spectrum-css/toast/dist/index-vars.css"
import Portal from "svelte-portal"
import { fly } from "svelte/transition"
import { Banner, BANNER_TYPES } from "@budibase/bbui"
import { licensing } from "stores/portal"
export let show = true
const oneDayInSeconds = 86400
$: license = $licensing.license
function daysUntilCancel() {
const cancelAt = license?.billing?.subscription?.cancelAt
const diffTime = Math.abs(cancelAt - new Date().getTime()) / 1000
return Math.floor(diffTime / oneDayInSeconds)
}
</script>
<Portal target=".banner-container">
<div class="banner">
{#if show}
<div transition:fly={{ y: -30 }}>
<Banner
type={BANNER_TYPES.INFO}
extraLinkText={"Please select a plan."}
extraLinkAction={$licensing.goToUpgradePage}
showCloseButton={false}
>
Your free trial will end in {daysUntilCancel()} days.
</Banner>
</div>
{/if}
</div>
</Portal>
<style>
.banner {
pointer-events: none;
width: 100%;
}
</style>

View File

@ -12,7 +12,7 @@ const defaultCacheFn = key => {
const upgradeAction = key => {
return defaultNavigateAction(
key,
"Upgrade Plan",
"Upgrade",
`${get(admin).accountPortalUrl}/portal/upgrade`
)
}

View File

@ -5,6 +5,7 @@
import { auth, licensing } from "stores/portal"
import { API } from "api"
import { PlanType } from "@budibase/types"
import { sdk } from "@budibase/shared-core"
let freeTrialModal
@ -14,7 +15,8 @@
const showFreeTrialModal = (planType, freeTrialModal) => {
if (
planType === PlanType.ENTERPRISE_BASIC_TRIAL &&
!$auth.user?.freeTrialConfirmedAt
!$auth.user?.freeTrialConfirmedAt &&
sdk.users.isAdmin($auth.user)
) {
freeTrialModal?.show()
}

View File

@ -98,14 +98,22 @@
})
}
async function fetchBackups(filters, page, dateRange) {
const response = await backups.searchBackups({
async function fetchBackups(filters, page, dateRange = []) {
const body = {
appId: $appStore.appId,
...filters,
page,
startDate: dateRange[0],
endDate: dateRange[1],
})
}
const [startDate, endDate] = dateRange
if (startDate) {
body.startDate = startDate
}
if (endDate) {
body.endDate = endDate
}
const response = await backups.searchBackups(body)
pageInfo.fetched(response.hasNextPage, response.nextPage)
// flatten so we have an easier structure to use for the table schema
@ -120,7 +128,7 @@
})
await fetchBackups(filterOpt, page)
notifications.success(response.message)
} catch {
} catch (err) {
notifications.error("Unable to create backup")
}
}

View File

@ -6,7 +6,7 @@
import { sdk } from "@budibase/shared-core"
</script>
{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan}
{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan && !$licensing.isEnterpriseTrial}
{#if $admin.cloud && $auth?.user?.accountPortalAccess}
<Button
cta

View File

@ -1,7 +1,7 @@
<script>
import { isActive, redirect, goto, url } from "@roxi/routify"
import { Icon, notifications, Tabs, Tab } from "@budibase/bbui"
import { organisation, auth, menu, appsStore } from "stores/portal"
import { organisation, auth, menu, appsStore, licensing } from "stores/portal"
import { onMount } from "svelte"
import UpgradeButton from "./_components/UpgradeButton.svelte"
import MobileMenu from "./_components/MobileMenu.svelte"
@ -10,6 +10,8 @@
import HelpMenu from "components/common/HelpMenu.svelte"
import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte"
import { sdk } from "@budibase/shared-core"
import EnterpriseBasicTrialBanner from "components/portal/licensing/EnterpriseBasicTrialBanner.svelte"
import { Constants } from "@budibase/frontend-core"
let loaded = false
let mobileMenuVisible = false
@ -33,6 +35,14 @@
const showMobileMenu = () => (mobileMenuVisible = true)
const hideMobileMenu = () => (mobileMenuVisible = false)
const showFreeTrialBanner = () => {
return (
$licensing.license?.plan?.type ===
Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
sdk.users.isAdmin($auth.user)
)
}
onMount(async () => {
// Prevent non-builders from accessing the portal
if ($auth.user) {
@ -58,6 +68,7 @@
<HelpMenu />
<div class="container">
<VerificationPromptBanner />
<EnterpriseBasicTrialBanner show={showFreeTrialBanner()} />
<div class="nav">
<div class="branding">
<Logo />

View File

@ -29,6 +29,7 @@
const manageUrl = `${$admin.accountPortalUrl}/portal/billing`
const WARN_USAGE = ["Queries", "Automations", "Rows", "Day Passes", "Users"]
const oneDayInSeconds = 86400
const EXCLUDE_QUOTAS = {
Queries: () => true,
@ -104,24 +105,17 @@
if (!timestamp) {
return
}
const now = new Date()
now.setHours(0)
now.setMinutes(0)
const thenDate = new Date(timestamp)
thenDate.setHours(0)
thenDate.setMinutes(0)
const difference = thenDate.getTime() - now
// return the difference in days
return (difference / (1000 * 3600 * 24)).toFixed(0)
const diffTime = Math.abs(timestamp - new Date().getTime()) / 1000
return Math.floor(diffTime / oneDayInSeconds)
}
const setTextRows = () => {
textRows = []
if (cancelAt && !usesInvoicing) {
if (plan?.type !== Constants.PlanType.ENTERPRISE_BASIC_TRIAL) {
textRows.push({ message: "Subscription has been cancelled" })
}
textRows.push({
message: `${getDaysRemaining(cancelAt)} days remaining`,
tooltip: new Date(cancelAt),

View File

@ -117,6 +117,19 @@ export async function validate(
})
}
function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.BOOLEAN) {
if (row[col.name] === 1) {
row[col.name] = true
} else if (row[col.name] === 0) {
row[col.name] = false
}
}
}
return row
}
export async function sqlOutputProcessing(
rows: DatasourcePlusQueryResponse,
table: Table,
@ -161,7 +174,9 @@ export async function sqlOutputProcessing(
if (thisRow._id == null) {
throw new Error("Unable to generate row ID for SQL rows")
}
finalRows[thisRow._id] = thisRow
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
// do this at end once its been added to the final rows
finalRows = await updateRelationshipColumns(
table,

View File

@ -4,10 +4,12 @@ import { APIError } from "@budibase/types"
describe("/api/applications/:appId/sync", () => {
let config = setup.getConfig()
afterAll(setup.afterAll)
beforeAll(async () => {
await config.init()
})
afterAll(async () => {
setup.afterAll()
})
describe("/api/attachments/process", () => {
it("should accept an image file upload", async () => {
@ -18,7 +20,8 @@ describe("/api/applications/:appId/sync", () => {
expect(resp.length).toBe(1)
let upload = resp[0]
expect(upload.url.endsWith(".jpg")).toBe(true)
expect(upload.url.split("?")[0].endsWith(".jpg")).toBe(true)
expect(upload.extension).toBe("jpg")
expect(upload.size).toBe(1)
expect(upload.name).toBe("1px.jpg")

View File

@ -1,16 +1,18 @@
import { mocks } from "@budibase/backend-core/tests"
import tk from "timekeeper"
import * as setup from "./utilities"
import { events } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import { mocks } from "@budibase/backend-core/tests"
mocks.licenses.useBackups()
describe("/backups", () => {
let config = setup.getConfig()
afterAll(setup.afterAll)
afterAll(async () => {
setup.afterAll()
})
beforeEach(async () => {
tk.reset()

View File

@ -856,7 +856,7 @@ describe.each([
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
return context.doInAppContext(config.getAppId(), async () => {
const enriched = await outputProcessing(table, [row])
expect((enriched as Row[])[0].attachment.url).toBe(
expect((enriched as Row[])[0].attachment.url.split("?")[0]).toBe(
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
)
})
@ -889,7 +889,7 @@ describe.each([
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
return context.doInAppContext(config.getAppId(), async () => {
const enriched = await outputProcessing(table, [row])
expect((enriched as Row[])[0].attachment[0].url).toBe(
expect((enriched as Row[])[0].attachment[0].url.split("?")[0]).toBe(
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
)
})

View File

@ -89,6 +89,22 @@ describe.each([
class SearchAssertion {
constructor(private readonly query: RowSearchParams) {}
private findRow(expectedRow: any, foundRows: any[]) {
const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
if (!row) {
const fields = Object.keys(expectedRow)
// To make the error message more readable, we only include the fields
// that are present in the expected row.
const searchedObjects = foundRows.map(row => _.pick(row, fields))
throw new Error(
`Failed to find row: ${JSON.stringify(
expectedRow
)} in ${JSON.stringify(searchedObjects)}`
)
}
return row
}
// Asserts that the query returns rows matching exactly the set of rows
// passed in. The order of the rows matters. Rows returned in an order
// different to the one passed in will cause the assertion to fail. Extra
@ -104,9 +120,7 @@ describe.each([
// eslint-disable-next-line jest/no-standalone-expect
expect(foundRows).toEqual(
expectedRows.map((expectedRow: any) =>
expect.objectContaining(
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
expect.objectContaining(this.findRow(expectedRow, foundRows))
)
)
}
@ -126,9 +140,7 @@ describe.each([
expect(foundRows).toEqual(
expect.arrayContaining(
expectedRows.map((expectedRow: any) =>
expect.objectContaining(
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
expect.objectContaining(this.findRow(expectedRow, foundRows))
)
)
)
@ -147,9 +159,7 @@ describe.each([
expect(foundRows).toEqual(
expect.arrayContaining(
expectedRows.map((expectedRow: any) =>
expect.objectContaining(
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
expect.objectContaining(this.findRow(expectedRow, foundRows))
)
)
)
@ -178,6 +188,67 @@ describe.each([
return expectSearch({ query })
}
describe("boolean", () => {
beforeAll(async () => {
await createTable({
isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
})
await createRows([{ isTrue: true }, { isTrue: false }])
})
describe("equal", () => {
it("successfully finds true row", () =>
expectQuery({ equal: { isTrue: true } }).toMatchExactly([
{ isTrue: true },
]))
it("successfully finds false row", () =>
expectQuery({ equal: { isTrue: false } }).toMatchExactly([
{ isTrue: false },
]))
})
describe("notEqual", () => {
it("successfully finds false row", () =>
expectQuery({ notEqual: { isTrue: true } }).toContainExactly([
{ isTrue: false },
]))
it("successfully finds true row", () =>
expectQuery({ notEqual: { isTrue: false } }).toContainExactly([
{ isTrue: true },
]))
})
describe("oneOf", () => {
it("successfully finds true row", () =>
expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([
{ isTrue: true },
]))
it("successfully finds false row", () =>
expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([
{ isTrue: false },
]))
})
describe("sort", () => {
it("sorts ascending", () =>
expectSearch({
query: {},
sort: "isTrue",
sortOrder: SortOrder.ASCENDING,
}).toMatchExactly([{ isTrue: false }, { isTrue: true }]))
it("sorts descending", () =>
expectSearch({
query: {},
sort: "isTrue",
sortOrder: SortOrder.DESCENDING,
}).toMatchExactly([{ isTrue: true }, { isTrue: false }]))
})
})
// Ensure all bindings resolve and perform as expected
describe("bindings", () => {
let globalUsers: any = []
@ -241,7 +312,7 @@ describe.each([
subtype: BBReferenceFieldSubType.USERS,
},
})
await createRows([...rows(config.getUser())])
await createRows(rows(config.getUser()))
})
// !! Current User is auto generated per run
@ -333,7 +404,7 @@ describe.each([
},
])
})
// TODO(samwho): fix for SQS
!isSqs &&
it("should match the session user id in a multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => {

View File

@ -1,3 +1,13 @@
// Directly mock the AWS SDK
jest.mock("aws-sdk", () => ({
S3: jest.fn(() => ({
getSignedUrl: jest.fn(
(operation, params) => `http://example.com/${params.Bucket}/${params.Key}`
),
upload: jest.fn(() => ({ Contents: {} })),
})),
}))
const setup = require("./utilities")
const { constants } = require("@budibase/backend-core")

View File

@ -1,14 +1,4 @@
// lucene searching not supported in test due to use of PouchDB
let rows: Row[] = []
jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
fullSearch: jest.fn(() => {
return {
rows,
}
}),
paginatedSearch: jest.fn(),
}))
import { Row, Table } from "@budibase/types"
import { Table } from "@budibase/types"
import * as setup from "./utilities"
const NAME = "Test"
@ -25,8 +15,8 @@ describe("Test a query step automation", () => {
description: "original description",
tableId: table._id,
}
rows.push(await config.createRow(row))
rows.push(await config.createRow(row))
await config.createRow(row)
await config.createRow(row)
})
afterAll(setup.afterAll)

View File

@ -150,6 +150,22 @@ function getTableName(table?: Table): string | undefined {
}
}
function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
if (Array.isArray(query)) {
return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
} else {
if (query.bindings) {
query.bindings = query.bindings.map(binding => {
if (typeof binding === "boolean") {
return binding ? 1 : 0
}
return binding
})
}
}
return query
}
class InternalBuilder {
private readonly client: string
@ -654,7 +670,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
if (opts?.disableBindings) {
return { sql: query.toString() }
} else {
return getNativeSql(query)
let native = getNativeSql(query)
if (sqlClient === SqlClient.SQL_LITE) {
native = convertBooleans(native)
}
return native
}
}

View File

@ -26,13 +26,13 @@ import { parse } from "content-disposition"
import path from "path"
import { Builder as XmlBuilder } from "xml2js"
const BodyTypes = {
NONE: "none",
FORM_DATA: "form",
XML: "xml",
ENCODED: "encoded",
JSON: "json",
TEXT: "text",
enum BodyType {
NONE = "none",
FORM_DATA = "form",
XML = "xml",
ENCODED = "encoded",
JSON = "json",
TEXT = "text",
}
const coreFields = {
@ -54,7 +54,7 @@ const coreFields = {
},
bodyType: {
type: DatasourceFieldType.STRING,
enum: Object.values(BodyTypes),
enum: Object.values(BodyType),
},
pagination: {
type: DatasourceFieldType.OBJECT,
@ -131,7 +131,10 @@ class RestIntegration implements IntegrationBase {
}
async parseResponse(response: any, pagination: PaginationConfig | null) {
let data, raw, headers, filename
let data: any[] | string | undefined,
raw: string | undefined,
headers: Record<string, string> = {},
filename: string | undefined
const contentType = response.headers.get("content-type") || ""
const contentDisposition = response.headers.get("content-disposition") || ""
@ -149,7 +152,7 @@ class RestIntegration implements IntegrationBase {
} else {
if (response.status === 204) {
data = []
raw = []
raw = ""
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
@ -162,16 +165,18 @@ class RestIntegration implements IntegrationBase {
raw = xmlResponse.rawXml
} else {
data = await response.text()
raw = data
raw = data as string
}
}
} catch (err) {
throw `Failed to parse response body: ${err}`
}
const size = formatBytes(
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
)
let contentLength: string = response.headers.get("content-length")
if (!contentLength && raw) {
contentLength = Buffer.byteLength(raw, "utf8").toString()
}
const size = formatBytes(contentLength || "0")
const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
headers = response.headers.raw()
for (let [key, value] of Object.entries(headers)) {
@ -255,7 +260,7 @@ class RestIntegration implements IntegrationBase {
if (!input.headers) {
input.headers = {}
}
if (bodyType === BodyTypes.NONE) {
if (bodyType === BodyType.NONE) {
return input
}
let error,
@ -283,11 +288,11 @@ class RestIntegration implements IntegrationBase {
}
switch (bodyType) {
case BodyTypes.TEXT:
case BodyType.TEXT:
// content type defaults to plaintext
input.body = string
break
case BodyTypes.ENCODED: {
case BodyType.ENCODED: {
const params = new URLSearchParams()
for (let [key, value] of Object.entries(object)) {
params.append(key, value as string)
@ -298,7 +303,7 @@ class RestIntegration implements IntegrationBase {
input.body = params
break
}
case BodyTypes.FORM_DATA: {
case BodyType.FORM_DATA: {
const form = new FormData()
for (let [key, value] of Object.entries(object)) {
form.append(key, value)
@ -309,14 +314,14 @@ class RestIntegration implements IntegrationBase {
input.body = form
break
}
case BodyTypes.XML:
case BodyType.XML:
if (object != null && Object.keys(object).length) {
string = new XmlBuilder().buildObject(object)
}
input.body = string
input.headers["Content-Type"] = "application/xml"
break
case BodyTypes.JSON:
case BodyType.JSON:
// if JSON error, throw it
if (error) {
throw "Invalid JSON for request body"

View File

@ -1,6 +1,3 @@
import fs from "fs"
import { join } from "path"
const response = (body: any, extra?: any) => () => ({
promise: () => body,
...extra,
@ -62,9 +59,7 @@ class S3 {
Body: "",
},
{
createReadStream: jest
.fn()
.mockReturnValue(fs.createReadStream(join(__dirname, "aws-sdk.ts"))),
createReadStream: jest.fn().mockReturnValue("stream"),
}
)
)

View File

@ -1,7 +1,6 @@
jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
import { default as DynamoDBIntegration } from "../dynamodb"
jest.mock("aws-sdk")
class TestConfiguration {
integration: any

View File

@ -28,7 +28,6 @@ jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
import { default as RestIntegration } from "../rest"
import { RestAuthType } from "@budibase/types"
import fetch from "node-fetch"
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
import { Readable } from "stream"
const FormData = require("form-data")
@ -246,13 +245,13 @@ describe("REST Integration", () => {
expect(output.extra.headers["content-type"]).toEqual("application/xml")
})
test.each(contentTypes)(
test.each([...contentTypes, undefined])(
"should not throw an error on 204 no content",
async contentType => {
const input = buildInput(undefined, null, contentType, 204)
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual([])
expect(output.extra.raw).toEqual([])
expect(output.extra.raw).toEqual("")
expect(output.info.code).toEqual(204)
expect(output.extra.headers["content-type"]).toEqual(contentType)
}
@ -627,15 +626,6 @@ describe("REST Integration", () => {
})
describe("File Handling", () => {
beforeAll(async () => {
jest.unmock("aws-sdk")
await objectStoreTestProviders.minio.start()
})
afterAll(async () => {
await objectStoreTestProviders.minio.stop()
})
it("uploads file to object store and returns signed URL", async () => {
const responseData = Buffer.from("teest file contnt")
const filename = "test.tar.gz"

View File

@ -1,7 +1,6 @@
jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
import { default as S3Integration } from "../s3"
jest.mock("aws-sdk")
class TestConfiguration {
integration: any

View File

@ -164,8 +164,8 @@ export async function search(
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql,
bindings = query.bindings
let sql = query.sql
let bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")

View File

@ -14,3 +14,6 @@ process.env.WORKER_URL = "http://localhost:10000"
process.env.COUCH_DB_PASSWORD = "budibase"
process.env.COUCH_DB_USER = "budibase"
process.env.JWT_SECRET = "jwtsecret"
process.env.MINIO_URL = "http://localhost"
process.env.MINIO_ACCESS_KEY = "budibase"
process.env.MINIO_SECRET_KEY = "budibase"

View File

@ -100,13 +100,13 @@ describe("rowProcessor - outputProcessing", () => {
}
const output = await outputProcessing(table, row, { squash: false })
expect(output.attach[0].url).toBe(
expect(output.attach[0].url?.split("?")[0]).toBe(
"/files/signed/prod-budi-app-assets/test.jpg"
)
row.attach[0].url = ""
const output2 = await outputProcessing(table, row, { squash: false })
expect(output2.attach[0].url).toBe(
expect(output2.attach[0].url?.split("?")[0]).toBe(
"/files/signed/prod-budi-app-assets/test.jpg"
)
@ -141,13 +141,13 @@ describe("rowProcessor - outputProcessing", () => {
}
const output = await outputProcessing(table, row, { squash: false })
expect(output.attach.url).toBe(
expect(output.attach.url?.split("?")[0]).toBe(
"/files/signed/prod-budi-app-assets/test.jpg"
)
row.attach.url = ""
const output2 = await outputProcessing(table, row, { squash: false })
expect(output2.attach.url).toBe(
expect(output2.attach?.url?.split("?")[0]).toBe(
"/files/signed/prod-budi-app-assets/test.jpg"
)

View File

@ -2,7 +2,6 @@ jest.unmock("node-fetch")
jest.unmock("aws-sdk")
import { TestConfiguration } from "../../../../tests"
import { EmailTemplatePurpose } from "../../../../constants"
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
import { objectStore } from "@budibase/backend-core"
import tk from "timekeeper"
import { EmailAttachment } from "@budibase/types"
@ -19,12 +18,10 @@ describe("/api/global/email", () => {
beforeAll(async () => {
tk.reset()
await objectStoreTestProviders.minio.start()
await config.beforeAll()
})
afterAll(async () => {
await objectStoreTestProviders.minio.stop()
await config.afterAll()
})

View File

@ -2003,9 +2003,9 @@
regenerator-runtime "^0.14.0"
"@babel/runtime@^7.13.10":
version "7.24.4"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.4.tgz#de795accd698007a66ba44add6cc86542aff1edd"
integrity sha512-dkxf7+hn8mFBwKjs9bvBlArzLVxVbS8usaPUDd5p2a9JCL9tB8OaOVN1isD4+Xyk4ns89/xeOmbQvgdK7IIVdA==
version "7.24.5"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.5.tgz#230946857c053a36ccc66e1dd03b17dd0c4ed02c"
integrity sha512-Nms86NXrsaeU9vbBJKni6gXiEXZ4CVpYVzEjDH9Sb8vmZ3UljyA1GSOJl/6LGPO8EHLuSF9H+IxNXHPX8QHJ4g==
dependencies:
regenerator-runtime "^0.14.0"
@ -2769,13 +2769,13 @@
yargs "^17.7.2"
"@grpc/proto-loader@^0.7.8":
version "0.7.12"
resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.12.tgz#787b58e3e3771df30b1567c057b6ab89e3a42911"
integrity sha512-DCVwMxqYzpUCiDMl7hQ384FqP4T3DbNpXU8pt681l3UWCip1WUiD5JrkImUwCB9a7f2cq4CUTmi5r/xIMRPY1Q==
version "0.7.13"
resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.13.tgz#f6a44b2b7c9f7b609f5748c6eac2d420e37670cf"
integrity sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==
dependencies:
lodash.camelcase "^4.3.0"
long "^5.0.0"
protobufjs "^7.2.4"
protobufjs "^7.2.5"
yargs "^17.7.2"
"@hapi/hoek@^9.0.0":
@ -5746,9 +5746,9 @@
integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
"@types/node@>=8.1.0":
version "20.12.4"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.4.tgz#af5921bd75ccdf3a3d8b3fa75bf3d3359268cd11"
integrity sha512-E+Fa9z3wSQpzgYQdYmme5X3OTuejnnTx88A6p6vkkJosR3KBz+HpE3kqNm98VE6cfLFcISx7zW7MsJkH6KwbTw==
version "20.12.10"
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.10.tgz#8f0c3f12b0f075eee1fe20c1afb417e9765bef76"
integrity sha512-Eem5pH9pmWBHoGAT8Dr5fdc5rYA+4NAovdM4EktRPVAAiJhmWWfQrA0cFhAbOsQdSfIHjAud6YdkbL69+zSKjw==
dependencies:
undici-types "~5.26.4"
@ -5767,9 +5767,9 @@
undici-types "~5.26.4"
"@types/nodemailer@^6.4.4":
version "6.4.14"
resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
integrity sha512-fUWthHO9k9DSdPCSPRqcu6TWhYyxTBg382vlNIttSe9M7XfsT06y0f24KHXtbnijPGGRIcVvdKHTNikOI6qiHA==
version "6.4.15"
resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.15.tgz#494be695e11c438f7f5df738fb4ab740312a6ed2"
integrity sha512-0EBJxawVNjPkng1zm2vopRctuWVCxk34JcIlRuXSf54habUWdz1FB7wHDqOqvDa8Mtpt0Q3LTXQkAs2LNyK5jQ==
dependencies:
"@types/node" "*"
@ -5809,9 +5809,9 @@
"@types/passport-oauth2" "*"
"@types/passport-oauth2@*":
version "1.4.15"
resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.15.tgz#34f2684f53aad36e664cd01ca9879224229f47e7"
integrity sha512-9cUTP/HStNSZmhxXGuRrBJfEWzIEJRub2eyJu3CvkA+8HAMc9W3aKdFhVq+Qz1hi42qn+GvSAnz3zwacDSYWpw==
version "1.4.16"
resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.16.tgz#59189a9d69783a63d7fb92d19cd28f96c95740af"
integrity sha512-Sdr0rpAdkiidUOtyaapGgvXyMjqYlMTFHRy7gtJtzr0/ysEIa72N3j2FSHIRc14h29g1+dzDl8IW2WT2Mu29vQ==
dependencies:
"@types/express" "*"
"@types/oauth" "*"
@ -9404,11 +9404,16 @@ dateformat@^4.6.3:
resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
dayjs@^1.10.8, dayjs@^1.8.15:
dayjs@^1.10.8:
version "1.11.10"
resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0"
integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==
dayjs@^1.8.15:
version "1.11.11"
resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e"
integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==
dc-polyfill@^0.1.2:
version "0.1.3"
resolved "https://registry.yarnpkg.com/dc-polyfill/-/dc-polyfill-0.1.3.tgz#fe9eefc86813439dd46d6f9ad9582ec079c39720"
@ -10269,9 +10274,9 @@ ee-first@1.1.1:
integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==
ejs@^3.1.7:
version "3.1.9"
resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.9.tgz#03c9e8777fe12686a9effcef22303ca3d8eeb361"
integrity sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==
version "3.1.10"
resolved "https://registry.yarnpkg.com/ejs/-/ejs-3.1.10.tgz#69ab8358b14e896f80cc39e62087b88500c3ac3b"
integrity sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==
dependencies:
jake "^10.8.5"
@ -18310,9 +18315,9 @@ posthog-js@^1.118.0:
preact "^10.19.3"
posthog-js@^1.13.4:
version "1.103.1"
resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.103.1.tgz#f846c413c28aca204dc1527f49d39f651348f3c4"
integrity sha512-cFXFU4Z4kl/+RUUV4ju1DlfM7dwCGi6H9xWsfhljIhGcBbT8UfS4JGgZGXl9ABQDdgDPb9xciqnysFSsUQshTA==
version "1.131.3"
resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.131.3.tgz#bd3e6123dc715f089825a92d3ec62480b7ec0a76"
integrity sha512-ds/TADDS+rT/WgUyeW4cJ+X+fX+O1KdkOyssNI/tP90PrFf0IJsck5B42YOLhfz87U2vgTyBaKHkdlMgWuOFog==
dependencies:
fflate "^0.4.8"
preact "^10.19.3"
@ -18954,9 +18959,9 @@ q@^1.1.2:
integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==
qs@^6.10.3:
version "6.12.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.0.tgz#edd40c3b823995946a8a0b1f208669c7a200db77"
integrity sha512-trVZiI6RMOkO476zLGaBIzszOdFPnCCXHPG9kn0yuS1uz6xdVxPfZdB3vUig9pxPFDM9BRAgz/YUIVQ1/vuiUg==
version "6.12.1"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.1.tgz#39422111ca7cbdb70425541cba20c7d7b216599a"
integrity sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==
dependencies:
side-channel "^1.0.6"
@ -20735,16 +20740,7 @@ string-similarity@^4.0.4:
resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"
"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@ -20834,7 +20830,7 @@ stringify-object@^3.2.1:
is-obj "^1.0.1"
is-regexp "^1.0.0"
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@ -20848,13 +20844,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
dependencies:
ansi-regex "^4.1.0"
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-ansi@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@ -22777,7 +22766,7 @@ worker-farm@1.7.0:
dependencies:
errno "~0.1.7"
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@ -22795,15 +22784,6 @@ wrap-ansi@^5.1.0:
string-width "^3.0.0"
strip-ansi "^5.0.0"
wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"
wrap-ansi@^8.1.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"