Merge branch 'master' of github.com:Budibase/budibase into feature/sqs-table-cleanup
This commit is contained in:
mike12345567
commit
fd7f6455bd
|
|
@ -281,7 +281,7 @@ export function doInScimContext(task: any) {
|
|||
return newContext(updates, task)
|
||||
}
|
||||
|
||||
export async function ensureSnippetContext() {
|
||||
export async function ensureSnippetContext(enabled = !env.isTest()) {
|
||||
const ctx = getCurrentContext()
|
||||
|
||||
// If we've already added snippets to context, continue
|
||||
|
|
@ -292,7 +292,7 @@ export async function ensureSnippetContext() {
|
|||
// Otherwise get snippets for this app and update context
|
||||
let snippets: Snippet[] | undefined
|
||||
const db = getAppDB()
|
||||
if (db && !env.isTest()) {
|
||||
if (db && enabled) {
|
||||
const app = await db.get<App>(DocumentType.APP_METADATA)
|
||||
snippets = app.snippets
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
|
|||
import { v4 } from "uuid"
|
||||
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
|
||||
import fsp from "fs/promises"
|
||||
import { HeadObjectOutput } from "aws-sdk/clients/s3"
|
||||
|
||||
const streamPipeline = promisify(stream.pipeline)
|
||||
// use this as a temporary store of buckets that are being created
|
||||
const STATE = {
|
||||
bucketCreationPromises: {},
|
||||
}
|
||||
const signedFilePrefix = "/files/signed"
|
||||
export const SIGNED_FILE_PREFIX = "/files/signed"
|
||||
|
||||
type ListParams = {
|
||||
ContinuationToken?: string
|
||||
|
|
@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & {
|
|||
path?: string | PathLike
|
||||
}
|
||||
|
||||
type StreamUploadParams = BaseUploadParams & {
|
||||
stream: ReadStream
|
||||
export type StreamTypes =
|
||||
| ReadStream
|
||||
| NodeJS.ReadableStream
|
||||
| ReadableStream<Uint8Array>
|
||||
|
||||
export type StreamUploadParams = BaseUploadParams & {
|
||||
stream?: StreamTypes
|
||||
}
|
||||
|
||||
const CONTENT_TYPE_MAP: any = {
|
||||
|
|
@ -174,11 +180,9 @@ export async function upload({
|
|||
const objectStore = ObjectStore(bucketName)
|
||||
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
|
||||
|
||||
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
|
||||
if (ttl && bucketCreated.created) {
|
||||
let ttlConfig = bucketTTLConfig(bucketName, ttl)
|
||||
if (objectStore.putBucketLifecycleConfiguration) {
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
|
||||
let contentType = type
|
||||
|
|
@ -222,11 +226,9 @@ export async function streamUpload({
|
|||
const objectStore = ObjectStore(bucketName)
|
||||
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
|
||||
|
||||
if (ttl && (bucketCreated.created || bucketCreated.exists)) {
|
||||
if (ttl && bucketCreated.created) {
|
||||
let ttlConfig = bucketTTLConfig(bucketName, ttl)
|
||||
if (objectStore.putBucketLifecycleConfiguration) {
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
|
||||
}
|
||||
|
||||
// Set content type for certain known extensions
|
||||
|
|
@ -333,7 +335,7 @@ export function getPresignedUrl(
|
|||
const signedUrl = new URL(url)
|
||||
const path = signedUrl.pathname
|
||||
const query = signedUrl.search
|
||||
return `${signedFilePrefix}${path}${query}`
|
||||
return `${SIGNED_FILE_PREFIX}${path}${query}`
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -521,6 +523,26 @@ export async function getReadStream(
|
|||
return client.getObject(params).createReadStream()
|
||||
}
|
||||
|
||||
export async function getObjectMetadata(
|
||||
bucket: string,
|
||||
path: string
|
||||
): Promise<HeadObjectOutput> {
|
||||
bucket = sanitizeBucket(bucket)
|
||||
path = sanitizeKey(path)
|
||||
|
||||
const client = ObjectStore(bucket)
|
||||
const params = {
|
||||
Bucket: bucket,
|
||||
Key: path,
|
||||
}
|
||||
|
||||
try {
|
||||
return await client.headObject(params).promise()
|
||||
} catch (err: any) {
|
||||
throw new Error("Unable to retrieve metadata from object")
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
|
||||
the bucket and the path from it
|
||||
|
|
@ -530,7 +552,9 @@ export function extractBucketAndPath(
|
|||
): { bucket: string; path: string } | null {
|
||||
const baseUrl = url.split("?")[0]
|
||||
|
||||
const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
|
||||
const regex = new RegExp(
|
||||
`^${SIGNED_FILE_PREFIX}/(?<bucket>[^/]+)/(?<path>.+)$`
|
||||
)
|
||||
const match = baseUrl.match(regex)
|
||||
|
||||
if (match && match.groups) {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
import { join } from "path"
|
||||
import path, { join } from "path"
|
||||
import { tmpdir } from "os"
|
||||
import fs from "fs"
|
||||
import env from "../environment"
|
||||
import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
|
||||
|
||||
import * as objectStore from "./objectStore"
|
||||
import {
|
||||
AutomationAttachment,
|
||||
AutomationAttachmentContent,
|
||||
BucketedContent,
|
||||
} from "@budibase/types"
|
||||
/****************************************************
|
||||
* NOTE: When adding a new bucket - name *
|
||||
* sure that S3 usages (like budibase-infra) *
|
||||
|
|
@ -55,3 +60,50 @@ export const bucketTTLConfig = (
|
|||
|
||||
return params
|
||||
}
|
||||
|
||||
async function processUrlAttachment(
|
||||
attachment: AutomationAttachment
|
||||
): Promise<AutomationAttachmentContent> {
|
||||
const response = await fetch(attachment.url)
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Unexpected response ${response.statusText}`)
|
||||
}
|
||||
const fallbackFilename = path.basename(new URL(attachment.url).pathname)
|
||||
return {
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: response.body,
|
||||
}
|
||||
}
|
||||
|
||||
export async function processObjectStoreAttachment(
|
||||
attachment: AutomationAttachment
|
||||
): Promise<BucketedContent> {
|
||||
const result = objectStore.extractBucketAndPath(attachment.url)
|
||||
|
||||
if (result === null) {
|
||||
throw new Error("Invalid signed URL")
|
||||
}
|
||||
|
||||
const { bucket, path: objectPath } = result
|
||||
const readStream = await objectStore.getReadStream(bucket, objectPath)
|
||||
const fallbackFilename = path.basename(objectPath)
|
||||
return {
|
||||
bucket,
|
||||
path: objectPath,
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: readStream,
|
||||
}
|
||||
}
|
||||
|
||||
export async function processAutomationAttachment(
|
||||
attachment: AutomationAttachment
|
||||
): Promise<AutomationAttachmentContent | BucketedContent> {
|
||||
const isFullyFormedUrl =
|
||||
attachment.url?.startsWith("http://") ||
|
||||
attachment.url?.startsWith("https://")
|
||||
if (isFullyFormedUrl) {
|
||||
return await processUrlAttachment(attachment)
|
||||
} else {
|
||||
return await processObjectStoreAttachment(attachment)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -358,7 +358,8 @@
|
|||
value.customType !== "cron" &&
|
||||
value.customType !== "triggerSchema" &&
|
||||
value.customType !== "automationFields" &&
|
||||
value.type !== "attachment"
|
||||
value.type !== "attachment" &&
|
||||
value.type !== "attachment_single"
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
import { tables } from "stores/builder"
|
||||
import { Select, Checkbox, Label } from "@budibase/bbui"
|
||||
import { createEventDispatcher } from "svelte"
|
||||
import { FieldType } from "@budibase/types"
|
||||
|
||||
import RowSelectorTypes from "./RowSelectorTypes.svelte"
|
||||
import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
|
||||
import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
|
||||
|
|
@ -14,7 +16,6 @@
|
|||
export let bindings
|
||||
export let isTestModal
|
||||
export let isUpdateRow
|
||||
|
||||
$: parsedBindings = bindings.map(binding => {
|
||||
let clone = Object.assign({}, binding)
|
||||
clone.icon = "ShareAndroid"
|
||||
|
|
@ -26,15 +27,19 @@
|
|||
|
||||
$: {
|
||||
table = $tables.list.find(table => table._id === value?.tableId)
|
||||
schemaFields = Object.entries(table?.schema ?? {})
|
||||
// surface the schema so the user can see it in the json
|
||||
schemaFields.map(([, schema]) => {
|
||||
|
||||
// Just sorting attachment types to the bottom here for a cleaner UX
|
||||
schemaFields = Object.entries(table?.schema ?? {}).sort(
|
||||
([, schemaA], [, schemaB]) =>
|
||||
(schemaA.type === "attachment") - (schemaB.type === "attachment")
|
||||
)
|
||||
|
||||
schemaFields.forEach(([, schema]) => {
|
||||
if (!schema.autocolumn && !value[schema.name]) {
|
||||
value[schema.name] = ""
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const onChangeTable = e => {
|
||||
value["tableId"] = e.detail
|
||||
dispatch("change", value)
|
||||
|
|
@ -114,10 +119,16 @@
|
|||
</div>
|
||||
{#if schemaFields.length}
|
||||
{#each schemaFields as [field, schema]}
|
||||
{#if !schema.autocolumn && schema.type !== "attachment"}
|
||||
<div class="schema-fields">
|
||||
{#if !schema.autocolumn}
|
||||
<div
|
||||
class:schema-fields={schema.type !== FieldType.ATTACHMENTS &&
|
||||
schema.type !== FieldType.ATTACHMENT_SINGLE}
|
||||
>
|
||||
<Label>{field}</Label>
|
||||
<div class="field-width">
|
||||
<div
|
||||
class:field-width={schema.type !== FieldType.ATTACHMENTS &&
|
||||
schema.type !== FieldType.ATTACHMENT_SINGLE}
|
||||
>
|
||||
{#if isTestModal}
|
||||
<RowSelectorTypes
|
||||
{isTestModal}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
<script>
|
||||
import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
|
||||
import { FieldType } from "@budibase/types"
|
||||
import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
|
||||
import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
|
||||
import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
|
||||
import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
|
||||
import Editor from "components/integration/QueryEditor.svelte"
|
||||
import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
|
||||
|
||||
export let onChange
|
||||
export let field
|
||||
|
|
@ -22,6 +24,27 @@
|
|||
function schemaHasOptions(schema) {
|
||||
return !!schema.constraints?.inclusion?.length
|
||||
}
|
||||
|
||||
const handleAttachmentParams = keyValuObj => {
|
||||
let params = {}
|
||||
|
||||
if (
|
||||
schema.type === FieldType.ATTACHMENT_SINGLE &&
|
||||
Object.keys(keyValuObj).length === 0
|
||||
) {
|
||||
return []
|
||||
}
|
||||
if (!Array.isArray(keyValuObj)) {
|
||||
keyValuObj = [keyValuObj]
|
||||
}
|
||||
|
||||
if (keyValuObj.length) {
|
||||
for (let param of keyValuObj) {
|
||||
params[param.url] = param.filename
|
||||
}
|
||||
}
|
||||
return params
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if schemaHasOptions(schema) && schema.type !== "array"}
|
||||
|
|
@ -77,6 +100,35 @@
|
|||
on:change={e => onChange(e, field)}
|
||||
useLabel={false}
|
||||
/>
|
||||
{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE}
|
||||
<div class="attachment-field-spacinng">
|
||||
<KeyValueBuilder
|
||||
on:change={e =>
|
||||
onChange(
|
||||
{
|
||||
detail:
|
||||
schema.type === FieldType.ATTACHMENT_SINGLE
|
||||
? e.detail.length > 0
|
||||
? { url: e.detail[0].name, filename: e.detail[0].value }
|
||||
: {}
|
||||
: e.detail.map(({ name, value }) => ({
|
||||
url: name,
|
||||
filename: value,
|
||||
})),
|
||||
},
|
||||
field
|
||||
)}
|
||||
object={handleAttachmentParams(value[field])}
|
||||
allowJS
|
||||
{bindings}
|
||||
keyBindings
|
||||
customButtonText={"Add attachment"}
|
||||
keyPlaceholder={"URL"}
|
||||
valuePlaceholder={"Filename"}
|
||||
actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE &&
|
||||
Object.keys(value[field]).length >= 1}
|
||||
/>
|
||||
</div>
|
||||
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
|
||||
<svelte:component
|
||||
this={isTestModal ? ModalBindableInput : DrawerBindableInput}
|
||||
|
|
@ -90,3 +142,10 @@
|
|||
title={schema.name}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.attachment-field-spacinng {
|
||||
margin-top: var(--spacing-s);
|
||||
margin-bottom: var(--spacing-l);
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
<script>
|
||||
import { createEventDispatcher } from "svelte"
|
||||
import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
|
||||
import { ActionButton, Drawer, DrawerContent, Button } from "@budibase/bbui"
|
||||
import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
|
||||
import { getUserBindings } from "dataBinding"
|
||||
import { makePropSafe } from "@budibase/string-templates"
|
||||
|
||||
export let schema
|
||||
export let filters
|
||||
|
|
@ -10,7 +12,7 @@
|
|||
|
||||
const dispatch = createEventDispatcher()
|
||||
|
||||
let modal
|
||||
let drawer
|
||||
|
||||
$: tempValue = filters || []
|
||||
$: schemaFields = Object.entries(schema || {}).map(
|
||||
|
|
@ -22,37 +24,53 @@
|
|||
|
||||
$: text = getText(filters)
|
||||
$: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
|
||||
|
||||
$: bindings = [
|
||||
{
|
||||
type: "context",
|
||||
runtimeBinding: `${makePropSafe("now")}`,
|
||||
readableBinding: `Date`,
|
||||
category: "Date",
|
||||
icon: "Date",
|
||||
display: {
|
||||
name: "Server date",
|
||||
},
|
||||
},
|
||||
...getUserBindings(),
|
||||
]
|
||||
const getText = filters => {
|
||||
const count = filters?.filter(filter => filter.field)?.length
|
||||
return count ? `Filter (${count})` : "Filter"
|
||||
}
|
||||
</script>
|
||||
|
||||
<ActionButton icon="Filter" quiet {disabled} on:click={modal.show} {selected}>
|
||||
<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
|
||||
{text}
|
||||
</ActionButton>
|
||||
<Modal bind:this={modal}>
|
||||
<ModalContent
|
||||
title="Filter"
|
||||
confirmText="Save"
|
||||
size="XL"
|
||||
onConfirm={() => dispatch("change", tempValue)}
|
||||
>
|
||||
<div class="wrapper">
|
||||
<FilterBuilder
|
||||
allowBindings={false}
|
||||
{filters}
|
||||
{schemaFields}
|
||||
datasource={{ type: "table", tableId }}
|
||||
on:change={e => (tempValue = e.detail)}
|
||||
/>
|
||||
</div>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
|
||||
<style>
|
||||
.wrapper :global(.main) {
|
||||
padding: 0;
|
||||
}
|
||||
</style>
|
||||
<Drawer
|
||||
bind:this={drawer}
|
||||
title="Filtering"
|
||||
on:drawerHide
|
||||
on:drawerShow
|
||||
forceModal
|
||||
>
|
||||
<Button
|
||||
cta
|
||||
slot="buttons"
|
||||
on:click={() => {
|
||||
dispatch("change", tempValue)
|
||||
drawer.hide()
|
||||
}}
|
||||
>
|
||||
Save
|
||||
</Button>
|
||||
<DrawerContent slot="body">
|
||||
<FilterBuilder
|
||||
{filters}
|
||||
{schemaFields}
|
||||
datasource={{ type: "table", tableId }}
|
||||
on:change={e => (tempValue = e.detail)}
|
||||
{bindings}
|
||||
/>
|
||||
</DrawerContent>
|
||||
</Drawer>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
readableToRuntimeBinding,
|
||||
runtimeToReadableBinding,
|
||||
} from "dataBinding"
|
||||
import { FieldType } from "@budibase/types"
|
||||
|
||||
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
|
||||
import { createEventDispatcher, setContext } from "svelte"
|
||||
|
|
@ -102,6 +103,8 @@
|
|||
longform: value => !isJSBinding(value),
|
||||
json: value => !isJSBinding(value),
|
||||
boolean: isValidBoolean,
|
||||
attachment: false,
|
||||
attachment_single: false,
|
||||
}
|
||||
|
||||
const isValid = value => {
|
||||
|
|
@ -116,7 +119,16 @@
|
|||
if (type === "json" && !isJSBinding(value)) {
|
||||
return "json-slot-icon"
|
||||
}
|
||||
if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
|
||||
if (
|
||||
![
|
||||
"string",
|
||||
"number",
|
||||
"bigint",
|
||||
"barcodeqr",
|
||||
"attachment",
|
||||
"attachment_single",
|
||||
].includes(type)
|
||||
) {
|
||||
return "slot-icon"
|
||||
}
|
||||
return ""
|
||||
|
|
@ -157,7 +169,7 @@
|
|||
{updateOnChange}
|
||||
/>
|
||||
{/if}
|
||||
{#if !disabled && type !== "formula"}
|
||||
{#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
|
||||
<div
|
||||
class={`icon ${getIconClass(value, type)}`}
|
||||
on:click={() => {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
export let customButtonText = null
|
||||
export let keyBindings = false
|
||||
export let allowJS = false
|
||||
export let actionButtonDisabled = false
|
||||
export let compare = (option, value) => option === value
|
||||
|
||||
let fields = Object.entries(object || {}).map(([name, value]) => ({
|
||||
|
|
@ -189,7 +190,14 @@
|
|||
{/if}
|
||||
{#if !readOnly && !noAddButton}
|
||||
<div>
|
||||
<ActionButton icon="Add" secondary thin outline on:click={addEntry}>
|
||||
<ActionButton
|
||||
disabled={actionButtonDisabled}
|
||||
icon="Add"
|
||||
secondary
|
||||
thin
|
||||
outline
|
||||
on:click={addEntry}
|
||||
>
|
||||
{#if customButtonText}
|
||||
{customButtonText}
|
||||
{:else}
|
||||
|
|
|
|||
|
|
@ -289,6 +289,7 @@
|
|||
OperatorOptions.ContainsAny.value,
|
||||
].includes(filter.operator)}
|
||||
disabled={filter.noValue}
|
||||
type={filter.valueType}
|
||||
/>
|
||||
{:else}
|
||||
<Input disabled />
|
||||
|
|
@ -325,8 +326,6 @@
|
|||
<style>
|
||||
.container {
|
||||
width: 100%;
|
||||
max-width: 1000px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.fields {
|
||||
display: grid;
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
import { createAPIClient } from "../api"
|
||||
|
||||
export let API = createAPIClient()
|
||||
|
||||
export let value = null
|
||||
export let disabled
|
||||
export let multiselect = false
|
||||
|
|
@ -23,12 +24,14 @@
|
|||
$: component = multiselect ? Multiselect : Select
|
||||
</script>
|
||||
|
||||
<svelte:component
|
||||
this={component}
|
||||
bind:value
|
||||
autocomplete
|
||||
{options}
|
||||
getOptionLabel={option => option.email}
|
||||
getOptionValue={option => option._id}
|
||||
{disabled}
|
||||
/>
|
||||
<div class="user-control">
|
||||
<svelte:component
|
||||
this={component}
|
||||
bind:value
|
||||
autocomplete
|
||||
{options}
|
||||
getOptionLabel={option => option.email}
|
||||
getOptionValue={option => option._id}
|
||||
{disabled}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import stream from "stream"
|
|||
import archiver from "archiver"
|
||||
|
||||
import { quotas } from "@budibase/pro"
|
||||
import { objectStore } from "@budibase/backend-core"
|
||||
import { objectStore, context } from "@budibase/backend-core"
|
||||
import * as internal from "./internal"
|
||||
import * as external from "./external"
|
||||
import { isExternalTableID } from "../../../integrations/utils"
|
||||
|
|
@ -198,8 +198,18 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
|
|||
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
|
||||
const tableId = utils.getTableId(ctx)
|
||||
|
||||
await context.ensureSnippetContext(true)
|
||||
|
||||
const enrichedQuery = await utils.enrichSearchContext(
|
||||
{ ...ctx.request.body.query },
|
||||
{
|
||||
user: sdk.users.getUserContextBindings(ctx.user),
|
||||
}
|
||||
)
|
||||
|
||||
const searchParams: RowSearchParams = {
|
||||
...ctx.request.body,
|
||||
query: enrichedQuery,
|
||||
tableId,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,12 +73,15 @@ export function basicProcessing({
|
|||
// filter the row down to what is actually the row (not joined)
|
||||
for (let field of Object.values(table.schema)) {
|
||||
const fieldName = field.name
|
||||
const value = extractFieldValue({
|
||||
let value = extractFieldValue({
|
||||
row,
|
||||
tableName: table.name,
|
||||
fieldName,
|
||||
isLinked,
|
||||
})
|
||||
if (value instanceof Buffer) {
|
||||
value = value.toString()
|
||||
}
|
||||
// all responses include "select col as table.col" so that overlaps are handled
|
||||
if (value != null) {
|
||||
thisRow[fieldName] = value
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
getInternalRowId,
|
||||
} from "./basic"
|
||||
import sdk from "../../../../sdk"
|
||||
|
||||
import { processStringSync } from "@budibase/string-templates"
|
||||
import validateJs from "validate.js"
|
||||
|
||||
validateJs.extend(validateJs.validators.datetime, {
|
||||
|
|
@ -204,3 +204,63 @@ export async function sqlOutputProcessing(
|
|||
export function isUserMetadataTable(tableId: string) {
|
||||
return tableId === InternalTables.USER_METADATA
|
||||
}
|
||||
|
||||
export async function enrichArrayContext(
|
||||
fields: any[],
|
||||
inputs = {},
|
||||
helpers = true
|
||||
): Promise<any[]> {
|
||||
const map: Record<string, any> = {}
|
||||
for (let index in fields) {
|
||||
map[index] = fields[index]
|
||||
}
|
||||
const output = await enrichSearchContext(map, inputs, helpers)
|
||||
const outputArray: any[] = []
|
||||
for (let [key, value] of Object.entries(output)) {
|
||||
outputArray[parseInt(key)] = value
|
||||
}
|
||||
return outputArray
|
||||
}
|
||||
|
||||
export async function enrichSearchContext(
|
||||
fields: Record<string, any>,
|
||||
inputs = {},
|
||||
helpers = true
|
||||
): Promise<Record<string, any>> {
|
||||
const enrichedQuery: Record<string, any> = {}
|
||||
if (!fields || !inputs) {
|
||||
return enrichedQuery
|
||||
}
|
||||
const parameters = { ...inputs }
|
||||
|
||||
if (Array.isArray(fields)) {
|
||||
return enrichArrayContext(fields, inputs, helpers)
|
||||
}
|
||||
|
||||
// enrich the fields with dynamic parameters
|
||||
for (let key of Object.keys(fields)) {
|
||||
if (fields[key] == null) {
|
||||
enrichedQuery[key] = null
|
||||
continue
|
||||
}
|
||||
if (typeof fields[key] === "object") {
|
||||
// enrich nested fields object
|
||||
enrichedQuery[key] = await enrichSearchContext(
|
||||
fields[key],
|
||||
parameters,
|
||||
helpers
|
||||
)
|
||||
} else if (typeof fields[key] === "string") {
|
||||
// enrich string value as normal
|
||||
enrichedQuery[key] = processStringSync(fields[key], parameters, {
|
||||
noEscaping: true,
|
||||
noHelpers: !helpers,
|
||||
escapeNewlines: true,
|
||||
})
|
||||
} else {
|
||||
enrichedQuery[key] = fields[key]
|
||||
}
|
||||
}
|
||||
|
||||
return enrichedQuery
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ import {
|
|||
} from "@budibase/types"
|
||||
import { dataFilters } from "@budibase/shared-core"
|
||||
import sdk from "../../../sdk"
|
||||
import { db } from "@budibase/backend-core"
|
||||
import { db, context } from "@budibase/backend-core"
|
||||
import { enrichSearchContext } from "./utils"
|
||||
|
||||
export async function searchView(
|
||||
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
|
||||
|
|
@ -56,10 +57,16 @@ export async function searchView(
|
|||
})
|
||||
}
|
||||
|
||||
await context.ensureSnippetContext(true)
|
||||
|
||||
const enrichedQuery = await enrichSearchContext(query, {
|
||||
user: sdk.users.getUserContextBindings(ctx.user),
|
||||
})
|
||||
|
||||
const searchOptions: RequiredKeys<SearchViewRowRequest> &
|
||||
RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
|
||||
tableId: view.tableId,
|
||||
query,
|
||||
query: enrichedQuery,
|
||||
fields: viewFields,
|
||||
...getSortOptions(body, view),
|
||||
limit: body.limit,
|
||||
|
|
|
|||
|
|
@ -32,8 +32,6 @@ import * as uuid from "uuid"
|
|||
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
|
||||
tk.freeze(timestamp)
|
||||
|
||||
jest.unmock("mssql")
|
||||
|
||||
describe.each([
|
||||
["internal", undefined],
|
||||
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
|
||||
|
|
@ -131,7 +129,13 @@ describe.each([
|
|||
|
||||
const assertRowUsage = async (expected: number) => {
|
||||
const usage = await getRowUsage()
|
||||
expect(usage).toBe(expected)
|
||||
|
||||
// Because our quota tracking is not perfect, we allow a 10% margin of
|
||||
// error. This is to account for the fact that parallel writes can result
|
||||
// in some quota updates getting lost. We don't have any need to solve this
|
||||
// right now, so we just allow for some error.
|
||||
expect(usage).toBeGreaterThan(expected * 0.9)
|
||||
expect(usage).toBeLessThan(expected * 1.1)
|
||||
}
|
||||
|
||||
const defaultRowFields = isInternal
|
||||
|
|
@ -194,39 +198,99 @@ describe.each([
|
|||
await assertRowUsage(rowUsage)
|
||||
})
|
||||
|
||||
it("increment row autoId per create row request", async () => {
|
||||
const rowUsage = await getRowUsage()
|
||||
isInternal &&
|
||||
it("increment row autoId per create row request", async () => {
|
||||
const rowUsage = await getRowUsage()
|
||||
|
||||
const newTable = await config.api.table.save(
|
||||
saveTableRequest({
|
||||
schema: {
|
||||
"Row ID": {
|
||||
name: "Row ID",
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
icon: "ri-magic-line",
|
||||
autocolumn: true,
|
||||
constraints: {
|
||||
type: "number",
|
||||
presence: true,
|
||||
numericality: {
|
||||
greaterThanOrEqualTo: "",
|
||||
lessThanOrEqualTo: "",
|
||||
const newTable = await config.api.table.save(
|
||||
saveTableRequest({
|
||||
schema: {
|
||||
"Row ID": {
|
||||
name: "Row ID",
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
icon: "ri-magic-line",
|
||||
autocolumn: true,
|
||||
constraints: {
|
||||
type: "number",
|
||||
presence: true,
|
||||
numericality: {
|
||||
greaterThanOrEqualTo: "",
|
||||
lessThanOrEqualTo: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
)
|
||||
|
||||
let previousId = 0
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const row = await config.api.row.save(newTable._id!, {})
|
||||
expect(row["Row ID"]).toBeGreaterThan(previousId)
|
||||
previousId = row["Row ID"]
|
||||
}
|
||||
await assertRowUsage(rowUsage + 10)
|
||||
})
|
||||
let previousId = 0
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const row = await config.api.row.save(newTable._id!, {})
|
||||
expect(row["Row ID"]).toBeGreaterThan(previousId)
|
||||
previousId = row["Row ID"]
|
||||
}
|
||||
await assertRowUsage(rowUsage + 10)
|
||||
})
|
||||
|
||||
isInternal &&
|
||||
it("should increment auto ID correctly when creating rows in parallel", async () => {
|
||||
const table = await config.api.table.save(
|
||||
saveTableRequest({
|
||||
schema: {
|
||||
"Row ID": {
|
||||
name: "Row ID",
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
icon: "ri-magic-line",
|
||||
autocolumn: true,
|
||||
constraints: {
|
||||
type: "number",
|
||||
presence: true,
|
||||
numericality: {
|
||||
greaterThanOrEqualTo: "",
|
||||
lessThanOrEqualTo: "",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
const sequence = Array(50)
|
||||
.fill(0)
|
||||
.map((_, i) => i + 1)
|
||||
|
||||
// This block of code is simulating users creating auto ID rows at the
|
||||
// same time. It's expected that this operation will sometimes return
|
||||
// a document conflict error (409), but the idea is to retry in those
|
||||
// situations. The code below does this a large number of times with
|
||||
// small, random delays between them to try and get through the list
|
||||
// as quickly as possible.
|
||||
await Promise.all(
|
||||
sequence.map(async () => {
|
||||
const attempts = 20
|
||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||
try {
|
||||
await config.api.row.save(table._id!, {})
|
||||
return
|
||||
} catch (e) {
|
||||
await new Promise(r => setTimeout(r, Math.random() * 15))
|
||||
}
|
||||
}
|
||||
throw new Error(`Failed to create row after ${attempts} attempts`)
|
||||
})
|
||||
)
|
||||
|
||||
const rows = await config.api.row.fetch(table._id!)
|
||||
expect(rows).toHaveLength(50)
|
||||
|
||||
// The main purpose of this test is to ensure that even under pressure,
|
||||
// we maintain data integrity. An auto ID column should hand out
|
||||
// monotonically increasing unique integers no matter what.
|
||||
const ids = rows.map(r => r["Row ID"])
|
||||
expect(ids).toEqual(expect.arrayContaining(sequence))
|
||||
})
|
||||
|
||||
isInternal &&
|
||||
it("row values are coerced", async () => {
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import { tableForDatasource } from "../../../tests/utilities/structures"
|
||||
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
|
||||
import { db as dbCore } from "@budibase/backend-core"
|
||||
|
||||
import * as setup from "./utilities"
|
||||
import {
|
||||
AutoFieldSubType,
|
||||
Datasource,
|
||||
EmptyFilterOption,
|
||||
BBReferenceFieldSubType,
|
||||
FieldType,
|
||||
RowSearchParams,
|
||||
SearchFilters,
|
||||
|
|
@ -13,10 +15,14 @@ import {
|
|||
SortType,
|
||||
Table,
|
||||
TableSchema,
|
||||
User,
|
||||
} from "@budibase/types"
|
||||
import _ from "lodash"
|
||||
import tk from "timekeeper"
|
||||
import { encodeJSBinding } from "@budibase/string-templates"
|
||||
|
||||
jest.unmock("mssql")
|
||||
const serverTime = new Date("2024-05-06T00:00:00.000Z")
|
||||
tk.freeze(serverTime)
|
||||
|
||||
describe.each([
|
||||
["lucene", undefined],
|
||||
|
|
@ -35,11 +41,25 @@ describe.each([
|
|||
let datasource: Datasource | undefined
|
||||
let table: Table
|
||||
|
||||
const snippets = [
|
||||
{
|
||||
name: "WeeksAgo",
|
||||
code: "return function (weeks) {\n const currentTime = new Date();\n currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n return currentTime.toISOString();\n}",
|
||||
},
|
||||
]
|
||||
|
||||
beforeAll(async () => {
|
||||
if (isSqs) {
|
||||
envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
|
||||
}
|
||||
await config.init()
|
||||
|
||||
if (config.app?.appId) {
|
||||
config.app = await config.api.application.update(config.app?.appId, {
|
||||
snippets,
|
||||
})
|
||||
}
|
||||
|
||||
if (dsProvider) {
|
||||
datasource = await config.createDatasource({
|
||||
datasource: await dsProvider,
|
||||
|
|
@ -227,6 +247,232 @@ describe.each([
|
|||
})
|
||||
})
|
||||
|
||||
// Ensure all bindings resolve and perform as expected
|
||||
describe("bindings", () => {
|
||||
let globalUsers: any = []
|
||||
|
||||
const future = new Date(serverTime.getTime())
|
||||
future.setDate(future.getDate() + 30)
|
||||
|
||||
const rows = (currentUser: User) => {
|
||||
return [
|
||||
{ name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
|
||||
{ name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
|
||||
{ name: currentUser.firstName, appointment: future.toISOString() },
|
||||
{ name: "serverDate", appointment: serverTime.toISOString() },
|
||||
{
|
||||
name: "single user, session user",
|
||||
single_user: JSON.stringify([currentUser]),
|
||||
},
|
||||
{
|
||||
name: "single user",
|
||||
single_user: JSON.stringify([globalUsers[0]]),
|
||||
},
|
||||
{
|
||||
name: "multi user",
|
||||
multi_user: JSON.stringify(globalUsers),
|
||||
},
|
||||
{
|
||||
name: "multi user with session user",
|
||||
multi_user: JSON.stringify([...globalUsers, currentUser]),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
// Set up some global users
|
||||
globalUsers = await Promise.all(
|
||||
Array(2)
|
||||
.fill(0)
|
||||
.map(async () => {
|
||||
const globalUser = await config.globalUser()
|
||||
const userMedataId = globalUser._id
|
||||
? dbCore.generateUserMetadataID(globalUser._id)
|
||||
: null
|
||||
return {
|
||||
_id: globalUser._id,
|
||||
_meta: userMedataId,
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
await createTable({
|
||||
name: { name: "name", type: FieldType.STRING },
|
||||
appointment: { name: "appointment", type: FieldType.DATETIME },
|
||||
single_user: {
|
||||
name: "single_user",
|
||||
type: FieldType.BB_REFERENCE,
|
||||
subtype: BBReferenceFieldSubType.USER,
|
||||
},
|
||||
multi_user: {
|
||||
name: "multi_user",
|
||||
type: FieldType.BB_REFERENCE,
|
||||
subtype: BBReferenceFieldSubType.USERS,
|
||||
},
|
||||
})
|
||||
await createRows(rows(config.getUser()))
|
||||
})
|
||||
|
||||
// !! Current User is auto generated per run
|
||||
it("should return all rows matching the session user firstname", async () => {
|
||||
await expectQuery({
|
||||
equal: { name: "{{ [user].firstName }}" },
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: config.getUser().firstName,
|
||||
appointment: future.toISOString(),
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("should parse the date binding and return all rows after the resolved value", async () => {
|
||||
await expectQuery({
|
||||
range: {
|
||||
appointment: {
|
||||
low: "{{ [now] }}",
|
||||
high: "9999-00-00T00:00:00.000Z",
|
||||
},
|
||||
},
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: config.getUser().firstName,
|
||||
appointment: future.toISOString(),
|
||||
},
|
||||
{ name: "serverDate", appointment: serverTime.toISOString() },
|
||||
])
|
||||
})
|
||||
|
||||
it("should parse the date binding and return all rows before the resolved value", async () => {
|
||||
await expectQuery({
|
||||
range: {
|
||||
appointment: {
|
||||
low: "0000-00-00T00:00:00.000Z",
|
||||
high: "{{ [now] }}",
|
||||
},
|
||||
},
|
||||
}).toContainExactly([
|
||||
{ name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
|
||||
{ name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
|
||||
{ name: "serverDate", appointment: serverTime.toISOString() },
|
||||
])
|
||||
})
|
||||
|
||||
it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => {
|
||||
const jsBinding = "return snippets.WeeksAgo();"
|
||||
const encodedBinding = encodeJSBinding(jsBinding)
|
||||
|
||||
await expectQuery({
|
||||
range: {
|
||||
appointment: {
|
||||
low: "0000-00-00T00:00:00.000Z",
|
||||
high: encodedBinding,
|
||||
},
|
||||
},
|
||||
}).toContainExactly([
|
||||
{ name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
|
||||
{ name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
|
||||
])
|
||||
})
|
||||
|
||||
it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
|
||||
const jsBinding =
|
||||
"const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
|
||||
const encodedBinding = encodeJSBinding(jsBinding)
|
||||
|
||||
await expectQuery({
|
||||
range: {
|
||||
appointment: {
|
||||
low: "0000-00-00T00:00:00.000Z",
|
||||
high: encodedBinding,
|
||||
},
|
||||
},
|
||||
}).toContainExactly([
|
||||
{ name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
|
||||
{ name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
|
||||
])
|
||||
})
|
||||
|
||||
it("should match a single user row by the session user id", async () => {
|
||||
await expectQuery({
|
||||
equal: { single_user: "{{ [user]._id }}" },
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: "single user, session user",
|
||||
single_user: [{ _id: config.getUser()._id }],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
// TODO(samwho): fix for SQS
|
||||
!isSqs &&
|
||||
it("should match the session user id in a multi user field", async () => {
|
||||
const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
|
||||
return { _id: user._id }
|
||||
})
|
||||
|
||||
await expectQuery({
|
||||
contains: { multi_user: ["{{ [user]._id }}"] },
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: "multi user with session user",
|
||||
multi_user: allUsers,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
// TODO(samwho): fix for SQS
|
||||
!isSqs &&
|
||||
it("should not match the session user id in a multi user field", async () => {
|
||||
await expectQuery({
|
||||
notContains: { multi_user: ["{{ [user]._id }}"] },
|
||||
notEmpty: { multi_user: true },
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: "multi user",
|
||||
multi_user: globalUsers.map((user: any) => {
|
||||
return { _id: user._id }
|
||||
}),
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => {
|
||||
await expectQuery({
|
||||
oneOf: {
|
||||
single_user: [
|
||||
"{{ default [user]._id '_empty_' }}",
|
||||
globalUsers[0]._id,
|
||||
],
|
||||
},
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: "single user, session user",
|
||||
single_user: [{ _id: config.getUser()._id }],
|
||||
},
|
||||
{
|
||||
name: "single user",
|
||||
single_user: [{ _id: globalUsers[0]._id }],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => {
|
||||
await expectQuery({
|
||||
oneOf: {
|
||||
single_user: [
|
||||
"{{ default [user]._idx '_empty_' }}",
|
||||
globalUsers[0]._id,
|
||||
],
|
||||
},
|
||||
}).toContainExactly([
|
||||
{
|
||||
name: "single user",
|
||||
single_user: [{ _id: globalUsers[0]._id }],
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
|
||||
beforeAll(async () => {
|
||||
await createTable({
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { context, events } from "@budibase/backend-core"
|
||||
import {
|
||||
AutoFieldSubType,
|
||||
Datasource,
|
||||
BBReferenceFieldSubType,
|
||||
Datasource,
|
||||
FieldType,
|
||||
INTERNAL_TABLE_SOURCE_ID,
|
||||
InternalTable,
|
||||
|
|
@ -149,58 +149,59 @@ describe.each([
|
|||
expect(res.name).toBeUndefined()
|
||||
})
|
||||
|
||||
it("updates only the passed fields", async () => {
|
||||
await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
|
||||
const table = await config.api.table.save(
|
||||
tableForDatasource(datasource, {
|
||||
schema: {
|
||||
autoId: {
|
||||
name: "id",
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
autocolumn: true,
|
||||
constraints: {
|
||||
type: "number",
|
||||
presence: false,
|
||||
isInternal &&
|
||||
it("updates only the passed fields", async () => {
|
||||
await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
|
||||
const table = await config.api.table.save(
|
||||
tableForDatasource(datasource, {
|
||||
schema: {
|
||||
autoId: {
|
||||
name: "id",
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
autocolumn: true,
|
||||
constraints: {
|
||||
type: "number",
|
||||
presence: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
const newName = generator.guid()
|
||||
|
||||
const updatedTable = await config.api.table.save({
|
||||
...table,
|
||||
name: newName,
|
||||
})
|
||||
)
|
||||
|
||||
const newName = generator.guid()
|
||||
let expected: Table = {
|
||||
...table,
|
||||
name: newName,
|
||||
_id: expect.any(String),
|
||||
}
|
||||
if (isInternal) {
|
||||
expected._rev = expect.stringMatching(/^2-.+/)
|
||||
}
|
||||
|
||||
const updatedTable = await config.api.table.save({
|
||||
...table,
|
||||
name: newName,
|
||||
expect(updatedTable).toEqual(expected)
|
||||
|
||||
const persistedTable = await config.api.table.get(updatedTable._id!)
|
||||
expected = {
|
||||
...table,
|
||||
name: newName,
|
||||
_id: updatedTable._id,
|
||||
}
|
||||
if (datasource?.isSQL) {
|
||||
expected.sql = true
|
||||
}
|
||||
if (isInternal) {
|
||||
expected._rev = expect.stringMatching(/^2-.+/)
|
||||
}
|
||||
expect(persistedTable).toEqual(expected)
|
||||
})
|
||||
|
||||
let expected: Table = {
|
||||
...table,
|
||||
name: newName,
|
||||
_id: expect.any(String),
|
||||
}
|
||||
if (isInternal) {
|
||||
expected._rev = expect.stringMatching(/^2-.+/)
|
||||
}
|
||||
|
||||
expect(updatedTable).toEqual(expected)
|
||||
|
||||
const persistedTable = await config.api.table.get(updatedTable._id!)
|
||||
expected = {
|
||||
...table,
|
||||
name: newName,
|
||||
_id: updatedTable._id,
|
||||
}
|
||||
if (datasource?.isSQL) {
|
||||
expected.sql = true
|
||||
}
|
||||
if (isInternal) {
|
||||
expected._rev = expect.stringMatching(/^2-.+/)
|
||||
}
|
||||
expect(persistedTable).toEqual(expected)
|
||||
})
|
||||
})
|
||||
|
||||
describe("user table", () => {
|
||||
isInternal &&
|
||||
|
|
@ -214,6 +215,57 @@ describe.each([
|
|||
})
|
||||
})
|
||||
|
||||
describe("external table validation", () => {
|
||||
!isInternal &&
|
||||
it("should error if column is of type auto", async () => {
|
||||
const table = basicTable(datasource)
|
||||
await config.api.table.save(
|
||||
{
|
||||
...table,
|
||||
schema: {
|
||||
...table.schema,
|
||||
auto: {
|
||||
name: "auto",
|
||||
autocolumn: true,
|
||||
type: FieldType.AUTO,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
status: 400,
|
||||
body: {
|
||||
message: `Column "auto" has type "${FieldType.AUTO}" - this is not supported.`,
|
||||
},
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
!isInternal &&
|
||||
it("should error if column has auto subtype", async () => {
|
||||
const table = basicTable(datasource)
|
||||
await config.api.table.save(
|
||||
{
|
||||
...table,
|
||||
schema: {
|
||||
...table.schema,
|
||||
auto: {
|
||||
name: "auto",
|
||||
autocolumn: true,
|
||||
type: FieldType.NUMBER,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
status: 400,
|
||||
body: {
|
||||
message: `Column "auto" has subtype "${AutoFieldSubType.AUTO_ID}" - this is not supported.`,
|
||||
},
|
||||
}
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
it("should add a new column for an internal DB table", async () => {
|
||||
const saveTableRequest: SaveTableRequest = {
|
||||
...basicTable(),
|
||||
|
|
|
|||
|
|
@ -24,8 +24,6 @@ import merge from "lodash/merge"
|
|||
import { quotas } from "@budibase/pro"
|
||||
import { roles } from "@budibase/backend-core"
|
||||
|
||||
jest.unmock("mssql")
|
||||
|
||||
describe.each([
|
||||
["internal", undefined],
|
||||
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
|
||||
|
|
|
|||
|
|
@ -4,8 +4,11 @@ import {
|
|||
encodeJSBinding,
|
||||
} from "@budibase/string-templates"
|
||||
import sdk from "../sdk"
|
||||
import { Row } from "@budibase/types"
|
||||
import { AutomationAttachment, FieldType, Row } from "@budibase/types"
|
||||
import { LoopInput, LoopStepType } from "../definitions/automations"
|
||||
import { objectStore, context } from "@budibase/backend-core"
|
||||
import * as uuid from "uuid"
|
||||
import path from "path"
|
||||
|
||||
/**
|
||||
* When values are input to the system generally they will be of type string as this is required for template strings.
|
||||
|
|
@ -96,6 +99,98 @@ export function getError(err: any) {
|
|||
return typeof err !== "string" ? err.toString() : err
|
||||
}
|
||||
|
||||
export async function sendAutomationAttachmentsToStorage(
|
||||
tableId: string,
|
||||
row: Row
|
||||
): Promise<Row> {
|
||||
const table = await sdk.tables.getTable(tableId)
|
||||
const attachmentRows: Record<
|
||||
string,
|
||||
AutomationAttachment[] | AutomationAttachment
|
||||
> = {}
|
||||
|
||||
for (const [prop, value] of Object.entries(row)) {
|
||||
const schema = table.schema[prop]
|
||||
if (
|
||||
schema?.type === FieldType.ATTACHMENTS ||
|
||||
schema?.type === FieldType.ATTACHMENT_SINGLE
|
||||
) {
|
||||
attachmentRows[prop] = value
|
||||
}
|
||||
}
|
||||
for (const [prop, attachments] of Object.entries(attachmentRows)) {
|
||||
if (Array.isArray(attachments)) {
|
||||
if (attachments.length) {
|
||||
row[prop] = await Promise.all(
|
||||
attachments.map(attachment => generateAttachmentRow(attachment))
|
||||
)
|
||||
}
|
||||
} else if (Object.keys(row[prop]).length > 0) {
|
||||
row[prop] = await generateAttachmentRow(attachments)
|
||||
}
|
||||
}
|
||||
|
||||
return row
|
||||
}
|
||||
|
||||
async function generateAttachmentRow(attachment: AutomationAttachment) {
|
||||
const prodAppId = context.getProdAppId()
|
||||
|
||||
async function uploadToS3(
|
||||
extension: string,
|
||||
content: objectStore.StreamTypes
|
||||
) {
|
||||
const fileName = `${uuid.v4()}${extension}`
|
||||
const s3Key = `${prodAppId}/attachments/${fileName}`
|
||||
|
||||
await objectStore.streamUpload({
|
||||
bucket: objectStore.ObjectStoreBuckets.APPS,
|
||||
stream: content,
|
||||
filename: s3Key,
|
||||
})
|
||||
|
||||
return s3Key
|
||||
}
|
||||
|
||||
async function getSize(s3Key: string) {
|
||||
return (
|
||||
await objectStore.getObjectMetadata(
|
||||
objectStore.ObjectStoreBuckets.APPS,
|
||||
s3Key
|
||||
)
|
||||
).ContentLength
|
||||
}
|
||||
|
||||
try {
|
||||
const { filename } = attachment
|
||||
const extension = path.extname(filename)
|
||||
const attachmentResult = await objectStore.processAutomationAttachment(
|
||||
attachment
|
||||
)
|
||||
|
||||
let s3Key = ""
|
||||
if (
|
||||
"path" in attachmentResult &&
|
||||
attachmentResult.path.startsWith(`${prodAppId}/attachments/`)
|
||||
) {
|
||||
s3Key = attachmentResult.path
|
||||
} else {
|
||||
s3Key = await uploadToS3(extension, attachmentResult.content)
|
||||
}
|
||||
|
||||
const size = await getSize(s3Key)
|
||||
|
||||
return {
|
||||
size,
|
||||
name: filename,
|
||||
extension,
|
||||
key: s3Key,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to process attachment:", error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
export function substituteLoopStep(hbsString: string, substitute: string) {
|
||||
let checkForJS = isJSBinding(hbsString)
|
||||
let substitutedHbsString = ""
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import { save } from "../../api/controllers/row"
|
||||
import { cleanUpRow, getError } from "../automationUtils"
|
||||
import {
|
||||
cleanUpRow,
|
||||
getError,
|
||||
sendAutomationAttachmentsToStorage,
|
||||
} from "../automationUtils"
|
||||
import { buildCtx } from "./utils"
|
||||
import {
|
||||
AutomationActionStepId,
|
||||
|
|
@ -89,6 +93,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
|
|||
|
||||
try {
|
||||
inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
|
||||
inputs.row = await sendAutomationAttachmentsToStorage(
|
||||
inputs.row.tableId,
|
||||
inputs.row
|
||||
)
|
||||
await save(ctx)
|
||||
return {
|
||||
row: inputs.row,
|
||||
|
|
|
|||
|
|
@ -108,7 +108,15 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
|
|||
|
||||
try {
|
||||
if (tableId) {
|
||||
inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row)
|
||||
inputs.row = await automationUtils.cleanUpRow(
|
||||
inputs.row.tableId,
|
||||
inputs.row
|
||||
)
|
||||
|
||||
inputs.row = await automationUtils.sendAutomationAttachmentsToStorage(
|
||||
inputs.row.tableId,
|
||||
inputs.row
|
||||
)
|
||||
}
|
||||
await rowController.patch(ctx)
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,18 @@
|
|||
import * as setup from "./utilities"
|
||||
import { basicTableWithAttachmentField } from "../../tests/utilities/structures"
|
||||
import { objectStore } from "@budibase/backend-core"
|
||||
|
||||
async function uploadTestFile(filename: string) {
|
||||
let bucket = "testbucket"
|
||||
await objectStore.upload({
|
||||
bucket,
|
||||
filename,
|
||||
body: Buffer.from("test data"),
|
||||
})
|
||||
let presignedUrl = await objectStore.getPresignedUrl(bucket, filename, 60000)
|
||||
|
||||
return presignedUrl
|
||||
}
|
||||
describe("test the create row action", () => {
|
||||
let table: any
|
||||
let row: any
|
||||
|
|
@ -43,4 +56,76 @@ describe("test the create row action", () => {
|
|||
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {})
|
||||
expect(res.success).toEqual(false)
|
||||
})
|
||||
|
||||
it("should check that an attachment field is sent to storage and parsed", async () => {
|
||||
let attachmentTable = await config.createTable(
|
||||
basicTableWithAttachmentField()
|
||||
)
|
||||
|
||||
let attachmentRow: any = {
|
||||
tableId: attachmentTable._id,
|
||||
}
|
||||
|
||||
let filename = "test1.txt"
|
||||
let presignedUrl = await uploadTestFile(filename)
|
||||
let attachmentObject = [
|
||||
{
|
||||
url: presignedUrl,
|
||||
filename,
|
||||
},
|
||||
]
|
||||
|
||||
attachmentRow.file_attachment = attachmentObject
|
||||
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
|
||||
row: attachmentRow,
|
||||
})
|
||||
|
||||
expect(res.success).toEqual(true)
|
||||
expect(res.row.file_attachment[0]).toHaveProperty("key")
|
||||
let s3Key = res.row.file_attachment[0].key
|
||||
|
||||
const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
|
||||
|
||||
const objectData = await client
|
||||
.headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
|
||||
.promise()
|
||||
|
||||
expect(objectData).toBeDefined()
|
||||
expect(objectData.ContentLength).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it("should check that an single attachment field is sent to storage and parsed", async () => {
|
||||
let attachmentTable = await config.createTable(
|
||||
basicTableWithAttachmentField()
|
||||
)
|
||||
|
||||
let attachmentRow: any = {
|
||||
tableId: attachmentTable._id,
|
||||
}
|
||||
|
||||
let filename = "test2.txt"
|
||||
let presignedUrl = await uploadTestFile(filename)
|
||||
let attachmentObject = {
|
||||
url: presignedUrl,
|
||||
filename,
|
||||
}
|
||||
|
||||
attachmentRow.single_file_attachment = attachmentObject
|
||||
const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
|
||||
row: attachmentRow,
|
||||
})
|
||||
|
||||
expect(res.success).toEqual(true)
|
||||
expect(res.row.single_file_attachment).toHaveProperty("key")
|
||||
let s3Key = res.row.single_file_attachment.key
|
||||
|
||||
const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
|
||||
|
||||
const objectData = await client
|
||||
.headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
|
||||
.promise()
|
||||
|
||||
expect(objectData).toBeDefined()
|
||||
expect(objectData.ContentLength).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1200,4 +1200,38 @@ describe("postgres integrations", () => {
|
|||
expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
|
||||
})
|
||||
})
|
||||
|
||||
describe("check custom column types", () => {
|
||||
beforeAll(async () => {
|
||||
await rawQuery(
|
||||
rawDatasource,
|
||||
`CREATE TABLE binaryTable (
|
||||
id BYTEA PRIMARY KEY,
|
||||
column1 TEXT,
|
||||
column2 INT
|
||||
);
|
||||
`
|
||||
)
|
||||
})
|
||||
|
||||
it("should handle binary columns", async () => {
|
||||
const response = await makeRequest(
|
||||
"post",
|
||||
`/api/datasources/${datasource._id}/schema`
|
||||
)
|
||||
expect(response.body).toBeDefined()
|
||||
expect(response.body.datasource.entities).toBeDefined()
|
||||
const table = response.body.datasource.entities["binarytable"]
|
||||
expect(table).toBeDefined()
|
||||
expect(table.schema.id.externalType).toBe("bytea")
|
||||
const row = await config.api.row.save(table._id, {
|
||||
id: "1111",
|
||||
column1: "hello",
|
||||
column2: 222,
|
||||
})
|
||||
expect(row._id).toBeDefined()
|
||||
const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
|
||||
expect(JSON.parse(decoded)[0]).toBe("1111")
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -139,13 +139,13 @@ class RestIntegration implements IntegrationBase {
|
|||
const contentType = response.headers.get("content-type") || ""
|
||||
const contentDisposition = response.headers.get("content-disposition") || ""
|
||||
if (
|
||||
contentDisposition.includes("filename") ||
|
||||
contentDisposition.includes("attachment") ||
|
||||
contentDisposition.includes("form-data")
|
||||
) {
|
||||
filename =
|
||||
path.basename(parse(contentDisposition).parameters?.filename) || ""
|
||||
}
|
||||
|
||||
try {
|
||||
if (filename) {
|
||||
return handleFileResponse(response, filename, this.startTimeMs)
|
||||
|
|
|
|||
|
|
@ -192,6 +192,11 @@ export function generateRowIdField(keyProps: any[] = []) {
|
|||
if (!Array.isArray(keyProps)) {
|
||||
keyProps = [keyProps]
|
||||
}
|
||||
for (let index in keyProps) {
|
||||
if (keyProps[index] instanceof Buffer) {
|
||||
keyProps[index] = keyProps[index].toString()
|
||||
}
|
||||
}
|
||||
// this conserves order and types
|
||||
// we have to swap the double quotes to single quotes for use in HBS statements
|
||||
// when using the literal helper the double quotes can break things
|
||||
|
|
|
|||
|
|
@ -1,117 +0,0 @@
|
|||
import {
|
||||
FieldType,
|
||||
Row,
|
||||
Table,
|
||||
RowSearchParams,
|
||||
INTERNAL_TABLE_SOURCE_ID,
|
||||
TableSourceType,
|
||||
} from "@budibase/types"
|
||||
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
|
||||
import { search } from "../internal"
|
||||
import {
|
||||
expectAnyInternalColsAttributes,
|
||||
generator,
|
||||
} from "@budibase/backend-core/tests"
|
||||
|
||||
describe("internal", () => {
|
||||
const config = new TestConfiguration()
|
||||
|
||||
const tableData: Table = {
|
||||
name: generator.word(),
|
||||
type: "table",
|
||||
sourceId: INTERNAL_TABLE_SOURCE_ID,
|
||||
sourceType: TableSourceType.INTERNAL,
|
||||
schema: {
|
||||
name: {
|
||||
name: "name",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
surname: {
|
||||
name: "surname",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
age: {
|
||||
name: "age",
|
||||
type: FieldType.NUMBER,
|
||||
constraints: {
|
||||
type: FieldType.NUMBER,
|
||||
},
|
||||
},
|
||||
address: {
|
||||
name: "address",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await config.init()
|
||||
})
|
||||
|
||||
describe("search", () => {
|
||||
const rows: Row[] = []
|
||||
beforeAll(async () => {
|
||||
await config.createTable(tableData)
|
||||
for (let i = 0; i < 10; i++) {
|
||||
rows.push(
|
||||
await config.createRow({
|
||||
name: generator.first(),
|
||||
surname: generator.last(),
|
||||
age: generator.age(),
|
||||
address: generator.address(),
|
||||
})
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
it("default search returns all the data", async () => {
|
||||
await config.doInContext(config.appId, async () => {
|
||||
const tableId = config.table!._id!
|
||||
|
||||
const searchParams: RowSearchParams = {
|
||||
tableId,
|
||||
query: {},
|
||||
}
|
||||
const result = await search(searchParams, config.table!)
|
||||
|
||||
expect(result.rows).toHaveLength(10)
|
||||
expect(result.rows).toEqual(
|
||||
expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
it("querying by fields will always return data attribute columns", async () => {
|
||||
await config.doInContext(config.appId, async () => {
|
||||
const tableId = config.table!._id!
|
||||
|
||||
const searchParams: RowSearchParams = {
|
||||
tableId,
|
||||
query: {},
|
||||
fields: ["name", "age"],
|
||||
}
|
||||
const result = await search(searchParams, config.table!)
|
||||
|
||||
expect(result.rows).toHaveLength(10)
|
||||
expect(result.rows).toEqual(
|
||||
expect.arrayContaining(
|
||||
rows.map(r => ({
|
||||
...expectAnyInternalColsAttributes,
|
||||
name: r.name,
|
||||
age: r.age,
|
||||
}))
|
||||
)
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,230 +0,0 @@
|
|||
import tk from "timekeeper"
|
||||
import * as internalSdk from "../internal"
|
||||
|
||||
import { generator } from "@budibase/backend-core/tests"
|
||||
import {
|
||||
INTERNAL_TABLE_SOURCE_ID,
|
||||
TableSourceType,
|
||||
FieldType,
|
||||
Table,
|
||||
AutoFieldSubType,
|
||||
AutoColumnFieldMetadata,
|
||||
} from "@budibase/types"
|
||||
|
||||
import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
|
||||
|
||||
tk.freeze(Date.now())
|
||||
|
||||
describe("sdk >> rows >> internal", () => {
|
||||
const config = new TestConfiguration()
|
||||
|
||||
beforeAll(async () => {
|
||||
await config.init()
|
||||
})
|
||||
|
||||
function makeRow() {
|
||||
return {
|
||||
name: generator.first(),
|
||||
surname: generator.last(),
|
||||
age: generator.age(),
|
||||
address: generator.address(),
|
||||
}
|
||||
}
|
||||
|
||||
describe("save", () => {
|
||||
const tableData: Table = {
|
||||
name: generator.word(),
|
||||
type: "table",
|
||||
sourceId: INTERNAL_TABLE_SOURCE_ID,
|
||||
sourceType: TableSourceType.INTERNAL,
|
||||
schema: {
|
||||
name: {
|
||||
name: "name",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
surname: {
|
||||
name: "surname",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
age: {
|
||||
name: "age",
|
||||
type: FieldType.NUMBER,
|
||||
constraints: {
|
||||
type: FieldType.NUMBER,
|
||||
},
|
||||
},
|
||||
address: {
|
||||
name: "address",
|
||||
type: FieldType.STRING,
|
||||
constraints: {
|
||||
type: FieldType.STRING,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it("save will persist the row properly", async () => {
|
||||
const table = await config.createTable(tableData)
|
||||
const row = makeRow()
|
||||
|
||||
await config.doInContext(config.appId, async () => {
|
||||
const response = await internalSdk.save(
|
||||
table._id!,
|
||||
row,
|
||||
config.getUser()._id
|
||||
)
|
||||
|
||||
expect(response).toEqual({
|
||||
table,
|
||||
row: {
|
||||
...row,
|
||||
type: "row",
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
},
|
||||
squashed: {
|
||||
...row,
|
||||
type: "row",
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
},
|
||||
})
|
||||
|
||||
const persistedRow = await config.api.row.get(
|
||||
table._id!,
|
||||
response.row._id!
|
||||
)
|
||||
expect(persistedRow).toEqual({
|
||||
...row,
|
||||
type: "row",
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
createdAt: expect.any(String),
|
||||
updatedAt: expect.any(String),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it("auto ids will update when creating new rows", async () => {
|
||||
const table = await config.createTable({
|
||||
...tableData,
|
||||
schema: {
|
||||
...tableData.schema,
|
||||
id: {
|
||||
name: "id",
|
||||
type: FieldType.AUTO,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
autocolumn: true,
|
||||
lastID: 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
const row = makeRow()
|
||||
|
||||
await config.doInContext(config.appId, async () => {
|
||||
const response = await internalSdk.save(
|
||||
table._id!,
|
||||
row,
|
||||
config.getUser()._id
|
||||
)
|
||||
|
||||
expect(response).toEqual({
|
||||
table: {
|
||||
...table,
|
||||
schema: {
|
||||
...table.schema,
|
||||
id: {
|
||||
...table.schema.id,
|
||||
lastID: 1,
|
||||
},
|
||||
},
|
||||
_rev: expect.stringMatching("2-.*"),
|
||||
},
|
||||
row: {
|
||||
...row,
|
||||
id: 1,
|
||||
type: "row",
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
},
|
||||
squashed: {
|
||||
...row,
|
||||
id: 1,
|
||||
type: "row",
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
},
|
||||
})
|
||||
|
||||
const persistedRow = await config.api.row.get(
|
||||
table._id!,
|
||||
response.row._id!
|
||||
)
|
||||
expect(persistedRow).toEqual({
|
||||
...row,
|
||||
type: "row",
|
||||
id: 1,
|
||||
_rev: expect.stringMatching("1-.*"),
|
||||
createdAt: expect.any(String),
|
||||
updatedAt: expect.any(String),
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
it("auto ids will update when creating new rows in parallel", async () => {
|
||||
function makeRows(count: number) {
|
||||
return Array.from({ length: count }, () => makeRow())
|
||||
}
|
||||
|
||||
const table = await config.createTable({
|
||||
...tableData,
|
||||
schema: {
|
||||
...tableData.schema,
|
||||
id: {
|
||||
name: "id",
|
||||
type: FieldType.AUTO,
|
||||
subtype: AutoFieldSubType.AUTO_ID,
|
||||
autocolumn: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await config.doInContext(config.appId, async () => {
|
||||
for (const row of makeRows(5)) {
|
||||
await internalSdk.save(table._id!, row, config.getUser()._id)
|
||||
}
|
||||
await Promise.all(
|
||||
makeRows(20).map(row =>
|
||||
internalSdk.save(table._id!, row, config.getUser()._id)
|
||||
)
|
||||
)
|
||||
for (const row of makeRows(5)) {
|
||||
await internalSdk.save(table._id!, row, config.getUser()._id)
|
||||
}
|
||||
})
|
||||
|
||||
const persistedRows = await config.getRows(table._id!)
|
||||
expect(persistedRows).toHaveLength(30)
|
||||
expect(persistedRows).toEqual(
|
||||
expect.arrayContaining(
|
||||
Array.from({ length: 30 }).map((_, i) =>
|
||||
expect.objectContaining({ id: i + 1 })
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
const persistedTable = await config.getTable(table._id)
|
||||
expect(
|
||||
(table.schema.id as AutoColumnFieldMetadata).lastID
|
||||
).toBeUndefined()
|
||||
expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
|
||||
30
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -6,6 +6,7 @@ import {
|
|||
Table,
|
||||
TableRequest,
|
||||
ViewV2,
|
||||
AutoFieldSubType,
|
||||
} from "@budibase/types"
|
||||
import { context } from "@budibase/backend-core"
|
||||
import { buildExternalTableId } from "../../../../integrations/utils"
|
||||
|
|
@ -29,6 +30,52 @@ import { populateExternalTableSchemas } from "../validation"
|
|||
import datasourceSdk from "../../datasources"
|
||||
import * as viewSdk from "../../views"
|
||||
|
||||
const DEFAULT_PRIMARY_COLUMN = "id"
|
||||
|
||||
function noPrimaryKey(table: Table) {
|
||||
return table.primary == null || table.primary.length === 0
|
||||
}
|
||||
|
||||
function validate(table: Table, oldTable?: Table) {
|
||||
if (
|
||||
!oldTable &&
|
||||
table.schema[DEFAULT_PRIMARY_COLUMN] &&
|
||||
noPrimaryKey(table)
|
||||
) {
|
||||
throw new Error(
|
||||
"External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
|
||||
)
|
||||
}
|
||||
|
||||
if (hasTypeChanged(table, oldTable)) {
|
||||
throw new Error("A column type has changed.")
|
||||
}
|
||||
|
||||
const autoSubTypes = Object.values(AutoFieldSubType)
|
||||
// check for auto columns, they are not allowed
|
||||
for (let [key, column] of Object.entries(table.schema)) {
|
||||
// this column is a special case, do not validate it
|
||||
if (key === DEFAULT_PRIMARY_COLUMN) {
|
||||
continue
|
||||
}
|
||||
// the auto-column type should never be used
|
||||
if (column.type === FieldType.AUTO) {
|
||||
throw new Error(
|
||||
`Column "${key}" has type "${FieldType.AUTO}" - this is not supported.`
|
||||
)
|
||||
}
|
||||
|
||||
if (
|
||||
column.subtype &&
|
||||
autoSubTypes.includes(column.subtype as AutoFieldSubType)
|
||||
) {
|
||||
throw new Error(
|
||||
`Column "${key}" has subtype "${column.subtype}" - this is not supported.`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function save(
|
||||
datasourceId: string,
|
||||
update: Table,
|
||||
|
|
@ -47,28 +94,18 @@ export async function save(
|
|||
oldTable = await getTable(tableId)
|
||||
}
|
||||
|
||||
if (
|
||||
!oldTable &&
|
||||
(tableToSave.primary == null || tableToSave.primary.length === 0)
|
||||
) {
|
||||
if (tableToSave.schema.id) {
|
||||
throw new Error(
|
||||
"External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
|
||||
)
|
||||
}
|
||||
// this will throw an error if something is wrong
|
||||
validate(tableToSave, oldTable)
|
||||
|
||||
tableToSave.primary = ["id"]
|
||||
tableToSave.schema.id = {
|
||||
if (!oldTable && noPrimaryKey(tableToSave)) {
|
||||
tableToSave.primary = [DEFAULT_PRIMARY_COLUMN]
|
||||
tableToSave.schema[DEFAULT_PRIMARY_COLUMN] = {
|
||||
type: FieldType.NUMBER,
|
||||
autocolumn: true,
|
||||
name: "id",
|
||||
name: DEFAULT_PRIMARY_COLUMN,
|
||||
}
|
||||
}
|
||||
|
||||
if (hasTypeChanged(tableToSave, oldTable)) {
|
||||
throw new Error("A column type has changed.")
|
||||
}
|
||||
|
||||
for (let view in tableToSave.views) {
|
||||
const tableView = tableToSave.views[view]
|
||||
if (!tableView || !viewSdk.isV2(tableView)) continue
|
||||
|
|
|
|||
|
|
@ -124,3 +124,12 @@ export async function syncGlobalUsers() {
|
|||
await db.bulkDocs(toWrite)
|
||||
}
|
||||
}
|
||||
|
||||
export function getUserContextBindings(user: ContextUser) {
|
||||
if (!user) {
|
||||
return {}
|
||||
}
|
||||
// Current user context for bindable search
|
||||
const { _id, _rev, firstName, lastName, email, status, roleId } = user
|
||||
return { _id, _rev, firstName, lastName, email, status, roleId }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,6 +81,12 @@ mocks.licenses.useUnlimited()
|
|||
|
||||
dbInit()
|
||||
|
||||
export interface CreateAppRequest {
|
||||
appName: string
|
||||
url?: string
|
||||
snippets?: any[]
|
||||
}
|
||||
|
||||
export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
|
||||
sourceId?: string
|
||||
sourceType?: TableSourceType
|
||||
|
|
@ -580,8 +586,6 @@ export default class TestConfiguration {
|
|||
|
||||
// APP
|
||||
async createApp(appName: string, url?: string): Promise<App> {
|
||||
// create dev app
|
||||
// clear any old app
|
||||
this.appId = undefined
|
||||
this.app = await context.doInTenant(
|
||||
this.tenantId!,
|
||||
|
|
@ -592,6 +596,7 @@ export default class TestConfiguration {
|
|||
})) as App
|
||||
)
|
||||
this.appId = this.app.appId
|
||||
|
||||
return await context.doInAppContext(this.app.appId!, async () => {
|
||||
// create production app
|
||||
this.prodApp = await this.publish()
|
||||
|
|
|
|||
|
|
@ -78,6 +78,32 @@ export function basicTable(
|
|||
)
|
||||
}
|
||||
|
||||
export function basicTableWithAttachmentField(
|
||||
datasource?: Datasource,
|
||||
...extra: Partial<Table>[]
|
||||
): Table {
|
||||
return tableForDatasource(
|
||||
datasource,
|
||||
{
|
||||
name: "TestTable",
|
||||
schema: {
|
||||
file_attachment: {
|
||||
type: FieldType.ATTACHMENTS,
|
||||
name: "description",
|
||||
constraints: {
|
||||
type: "array",
|
||||
},
|
||||
},
|
||||
single_file_attachment: {
|
||||
type: FieldType.ATTACHMENT_SINGLE,
|
||||
name: "description",
|
||||
},
|
||||
},
|
||||
},
|
||||
...extra
|
||||
)
|
||||
}
|
||||
|
||||
export function basicView(tableId: string) {
|
||||
return {
|
||||
tableId,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import * as linkRows from "../../db/linkedRows"
|
||||
import { processFormulas, fixAutoColumnSubType } from "./utils"
|
||||
import { context, objectStore, utils } from "@budibase/backend-core"
|
||||
import { objectStore, utils } from "@budibase/backend-core"
|
||||
import { InternalTables } from "../../db/utils"
|
||||
import { TYPE_TRANSFORM_MAP } from "./map"
|
||||
import {
|
||||
|
|
@ -25,45 +25,6 @@ type AutoColumnProcessingOpts = {
|
|||
noAutoRelationships?: boolean
|
||||
}
|
||||
|
||||
// Returns the next auto ID for a column in a table. On success, the table will
|
||||
// be updated which is why it gets returned. The nextID returned is guaranteed
|
||||
// to be given only to you, and if you don't use it it's gone forever (a gap
|
||||
// will be left in the auto ID sequence).
|
||||
//
|
||||
// This function can throw if it fails to generate an auto ID after so many
|
||||
// attempts.
|
||||
async function getNextAutoId(
|
||||
table: Table,
|
||||
column: string
|
||||
): Promise<{ table: Table; nextID: number }> {
|
||||
const db = context.getAppDB()
|
||||
for (let attempt = 0; attempt < 5; attempt++) {
|
||||
const schema = table.schema[column]
|
||||
if (schema.type !== FieldType.NUMBER && schema.type !== FieldType.AUTO) {
|
||||
throw new Error(`Column ${column} is not an auto column`)
|
||||
}
|
||||
schema.lastID = (schema.lastID || 0) + 1
|
||||
try {
|
||||
const resp = await db.put(table)
|
||||
table._rev = resp.rev
|
||||
return { table, nextID: schema.lastID }
|
||||
} catch (e: any) {
|
||||
if (e.status !== 409) {
|
||||
throw e
|
||||
}
|
||||
// We wait for a random amount of time before retrying. The randomness
|
||||
// makes it less likely for multiple requests modifying this table to
|
||||
// collide.
|
||||
await new Promise(resolve =>
|
||||
setTimeout(resolve, Math.random() * 1.2 ** attempt * 1000)
|
||||
)
|
||||
table = await db.get(table._id)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Failed to generate an auto ID")
|
||||
}
|
||||
|
||||
/**
|
||||
* This will update any auto columns that are found on the row/table with the correct information based on
|
||||
* time now and the current logged in user making the request.
|
||||
|
|
@ -116,9 +77,10 @@ export async function processAutoColumn(
|
|||
break
|
||||
case AutoFieldSubType.AUTO_ID:
|
||||
if (creating) {
|
||||
const { table: newTable, nextID } = await getNextAutoId(table, key)
|
||||
table = newTable
|
||||
row[key] = nextID
|
||||
schema.lastID = schema.lastID || 0
|
||||
row[key] = schema.lastID + 1
|
||||
schema.lastID++
|
||||
table.schema[key] = schema
|
||||
}
|
||||
break
|
||||
}
|
||||
|
|
@ -272,7 +234,7 @@ export async function outputProcessing<T extends Row[] | Row>(
|
|||
}
|
||||
} else if (column.type === FieldType.ATTACHMENT_SINGLE) {
|
||||
for (let row of enriched) {
|
||||
if (!row[property]) {
|
||||
if (!row[property] || Object.keys(row[property]).length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { Document } from "../document"
|
||||
import { EventEmitter } from "events"
|
||||
import { User } from "../global"
|
||||
import { ReadStream } from "fs"
|
||||
|
||||
export enum AutomationIOType {
|
||||
OBJECT = "object",
|
||||
|
|
@ -235,3 +236,18 @@ export interface AutomationMetadata extends Document {
|
|||
errorCount?: number
|
||||
automationChainCount?: number
|
||||
}
|
||||
|
||||
export type AutomationAttachment = {
|
||||
url: string
|
||||
filename: string
|
||||
}
|
||||
|
||||
export type AutomationAttachmentContent = {
|
||||
filename: string
|
||||
content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
|
||||
}
|
||||
|
||||
export type BucketedContent = AutomationAttachmentContent & {
|
||||
bucket: string
|
||||
path: string
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
jest.unmock("node-fetch")
|
||||
jest.unmock("aws-sdk")
|
||||
import { TestConfiguration } from "../../../../tests"
|
||||
import { EmailTemplatePurpose } from "../../../../constants"
|
||||
import { objectStore } from "@budibase/backend-core"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ import { processString } from "@budibase/string-templates"
|
|||
import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
|
||||
import { configs, cache, objectStore } from "@budibase/backend-core"
|
||||
import ical from "ical-generator"
|
||||
import fetch from "node-fetch"
|
||||
import path from "path"
|
||||
import _ from "lodash"
|
||||
|
||||
const nodemailer = require("nodemailer")
|
||||
|
||||
|
|
@ -165,39 +164,12 @@ export async function sendEmail(
|
|||
}),
|
||||
}
|
||||
if (opts?.attachments) {
|
||||
const attachments = await Promise.all(
|
||||
opts.attachments?.map(async attachment => {
|
||||
const isFullyFormedUrl =
|
||||
attachment.url.startsWith("http://") ||
|
||||
attachment.url.startsWith("https://")
|
||||
if (isFullyFormedUrl) {
|
||||
const response = await fetch(attachment.url)
|
||||
if (!response.ok) {
|
||||
throw new Error(`unexpected response ${response.statusText}`)
|
||||
}
|
||||
const fallbackFilename = path.basename(
|
||||
new URL(attachment.url).pathname
|
||||
)
|
||||
return {
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: response?.body,
|
||||
}
|
||||
} else {
|
||||
const url = attachment.url
|
||||
const result = objectStore.extractBucketAndPath(url)
|
||||
if (result === null) {
|
||||
throw new Error("Invalid signed URL")
|
||||
}
|
||||
const { bucket, path } = result
|
||||
const readStream = await objectStore.getReadStream(bucket, path)
|
||||
const fallbackFilename = path.split("/").pop() || ""
|
||||
return {
|
||||
filename: attachment.filename || fallbackFilename,
|
||||
content: readStream,
|
||||
}
|
||||
}
|
||||
})
|
||||
let attachments = await Promise.all(
|
||||
opts.attachments?.map(objectStore.processAutomationAttachment)
|
||||
)
|
||||
attachments = attachments.map(attachment => {
|
||||
return _.omit(attachment, "path")
|
||||
})
|
||||
message = { ...message, attachments }
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue