MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/signature-field-and-component

This commit is contained in:
Dean 2024-05-14 14:48:09 +01:00
parent b4af23229a 180c197054
commit 55e7ea55f7
11 changed files with 116 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.26.1",
"version": "2.26.2",
"npmClient": "yarn",
"packages": [
"packages/*",

12
packages/backend-core/src/objectStore/utils.ts
View File

@ -9,6 +9,9 @@ import {
AutomationAttachmentContent,
BucketedContent,
} from "@budibase/types"
import stream from "stream"
import streamWeb from "node:stream/web"
/****************************************************
* NOTE: When adding a new bucket - name *
* sure that S3 usages (like budibase-infra) *
@ -53,12 +56,10 @@ export const bucketTTLConfig = (
Rules: [lifecycleRule],
}
const params = {
return {
Bucket: bucketName,
LifecycleConfiguration: lifecycleConfiguration,
}
return params
}
async function processUrlAttachment(
@ -69,9 +70,12 @@ async function processUrlAttachment(
throw new Error(`Unexpected response ${response.statusText}`)
}
const fallbackFilename = path.basename(new URL(attachment.url).pathname)
if (!response.body) {
throw new Error("No response received for attachment")
}
return {
filename: attachment.filename || fallbackFilename,
content: response.body,
content: stream.Readable.fromWeb(response.body as streamWeb.ReadableStream),
}
}

19
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -375,6 +375,16 @@
return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
}
function handleAttachmentParams(keyValueObj) {
let params = {}
if (keyValueObj?.length) {
for (let param of keyValueObj) {
params[param.url] = param.filename
}
}
return params
}
onMount(async () => {
try {
await environment.loadVariables()
@ -382,15 +392,6 @@
console.error(error)
}
})
const handleAttachmentParams = keyValuObj => {
let params = {}
if (keyValuObj?.length) {
for (let param of keyValuObj) {
params[param.url] = param.filename
}
}
return params
}
</script>
<div class="fields">

12
packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte
View File

@ -25,22 +25,22 @@
return !!schema.constraints?.inclusion?.length
}
const handleAttachmentParams = keyValuObj => {
function handleAttachmentParams(keyValueObj) {
let params = {}
if (
(schema.type === FieldType.ATTACHMENT_SINGLE ||
schema.type === FieldType.SIGNATURE) &&
Object.keys(keyValuObj).length === 0
Object.keys(keyValueObj).length === 0
) {
return []
}
if (!Array.isArray(keyValuObj)) {
keyValuObj = [keyValuObj]
if (!Array.isArray(keyValueObj) && keyValueObj) {
keyValueObj = [keyValueObj]
}
if (keyValuObj.length) {
for (let param of keyValuObj) {
if (keyValueObj.length) {
for (let param of keyValueObj) {
params[param.url] = param.filename
}
}

4
packages/server/package.json
View File

@ -79,8 +79,7 @@
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-auth-library": "7.12.0",
"google-spreadsheet": "3.2.0",
"google-spreadsheet": "4.1.2",
"ioredis": "5.3.2",
"isolated-vm": "^4.7.2",
"jimp": "0.22.10",
@ -125,7 +124,6 @@
"@swc/jest": "0.2.27",
"@types/archiver": "6.0.2",
"@types/global-agent": "2.1.1",
"@types/google-spreadsheet": "3.1.5",
"@types/jest": "29.5.5",
"@types/koa": "2.13.4",
"@types/koa-send": "^4.1.6",

7
packages/server/src/automations/automationUtils.ts
View File

@ -164,7 +164,10 @@ async function generateAttachmentRow(attachment: AutomationAttachment) {
try {
const { filename } = attachment
const extension = path.extname(filename)
let extension = path.extname(filename)
if (extension.startsWith(".")) {
extension = extension.substring(1, extension.length)
}
const attachmentResult = await objectStore.processAutomationAttachment(
attachment
)
@ -183,8 +186,8 @@ async function generateAttachmentRow(attachment: AutomationAttachment) {
return {
size,
name: filename,
extension,
name: filename,
key: s3Key,
}
} catch (error) {

23
packages/server/src/automations/steps/updateRow.ts
View File

@ -94,18 +94,6 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
}
}
// have to clean up the row, remove the table from it
const ctx: any = buildCtx(appId, emitter, {
body: {
...inputs.row,
_id: inputs.rowId,
},
params: {
rowId: inputs.rowId,
tableId: tableId,
},
})
try {
if (tableId) {
inputs.row = await automationUtils.cleanUpRow(
@ -118,6 +106,17 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
inputs.row
)
}
// have to clean up the row, remove the table from it
const ctx: any = buildCtx(appId, emitter, {
body: {
...inputs.row,
_id: inputs.rowId,
},
params: {
rowId: inputs.rowId,
tableId: tableId,
},
})
await rowController.patch(ctx)
return {
row: ctx.body,

54
packages/server/src/integrations/googlesheets.ts
View File

@ -159,12 +159,12 @@ const SCHEMA: Integration = {
class GoogleSheetsIntegration implements DatasourcePlus {
private readonly config: GoogleSheetsConfig
private client: GoogleSpreadsheet
private readonly spreadsheetId: string
private client: GoogleSpreadsheet = undefined!
constructor(config: GoogleSheetsConfig) {
this.config = config
const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
this.client = new GoogleSpreadsheet(spreadsheetId)
this.spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId)
}
async testConnection(): Promise<ConnectionInfo> {
@ -192,7 +192,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
* @param spreadsheetId - the URL or standard spreadsheetId of the google sheet
* @returns spreadsheet Id of the google sheet
*/
cleanSpreadsheetUrl(spreadsheetId: string) {
private cleanSpreadsheetUrl(spreadsheetId: string) {
if (!spreadsheetId) {
throw new Error(
"You must set a spreadsheet ID in your configuration to fetch tables."
@ -202,7 +202,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return parts.length > 5 ? parts[5] : spreadsheetId
}
async fetchAccessToken(
private async fetchAccessToken(
payload: AuthTokenRequest
): Promise<AuthTokenResponse> {
const response = await fetch("https://www.googleapis.com/oauth2/v4/token", {
@ -227,7 +227,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return json
}
async connect() {
private async connect() {
try {
await setupCreationAuth(this.config)
@ -253,7 +253,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
access_token: tokenResponse.access_token,
})
this.client.useOAuth2Client(oauthClient)
this.client = new GoogleSpreadsheet(this.spreadsheetId, oauthClient)
await this.client.loadInfo()
} catch (err: any) {
// this happens for xlsx imports
@ -272,7 +272,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return sheets.map(s => s.title)
}
getTableSchema(
private getTableSchema(
title: string,
headerValues: string[],
datasourceId: string,
@ -386,18 +386,22 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
buildRowObject(headers: string[], values: string[], rowNumber: number) {
private buildRowObject(
headers: string[],
values: Record<string, string>,
rowNumber: number
) {
const rowObject: { rowNumber: number } & Row = {
rowNumber,
_id: rowNumber.toString(),
}
for (let i = 0; i < headers.length; i++) {
rowObject[headers[i]] = values[i]
rowObject[headers[i]] = values[headers[i]]
}
return rowObject
}
async createTable(name?: string) {
private async createTable(name?: string) {
if (!name) {
throw new Error("Must provide name for new sheet.")
}
@ -410,7 +414,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
async updateTable(table: TableRequest) {
private async updateTable(table: TableRequest) {
await this.connect()
const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow()
@ -457,7 +461,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
async deleteTable(sheet: any) {
private async deleteTable(sheet: any) {
try {
await this.connect()
const sheetToDelete = this.client.sheetsByTitle[sheet]
@ -476,7 +480,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
typeof query.row === "string" ? JSON.parse(query.row) : query.row
const row = await sheet.addRow(rowToInsert)
return [
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber),
]
} catch (err) {
console.error("Error writing to google sheets", err)
@ -484,7 +488,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
async createBulk(query: { sheet: string; rows: Row[] }) {
private async createBulk(query: { sheet: string; rows: Row[] }) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[query.sheet]
@ -494,7 +498,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
const rows = await sheet.addRows(rowsToInsert)
return rows.map(row =>
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber)
)
} catch (err) {
console.error("Error bulk writing to google sheets", err)
@ -549,7 +553,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
let response = []
for (let row of filtered) {
response.push(
this.buildRowObject(headerValues, row._rawData, row._rowNumber)
this.buildRowObject(headerValues, row.toObject(), row._rowNumber)
)
}
@ -599,10 +603,10 @@ class GoogleSheetsIntegration implements DatasourcePlus {
const updateValues =
typeof query.row === "string" ? JSON.parse(query.row) : query.row
for (let key in updateValues) {
row[key] = updateValues[key]
row.set(key, updateValues[key])
if (row[key] === null) {
row[key] = ""
if (row.get(key) === null) {
row.set(key, "")
}
const { type, subtype, constraints } = query.table.schema[key]
@ -610,13 +614,17 @@ class GoogleSheetsIntegration implements DatasourcePlus {
type === FieldType.BB_REFERENCE &&
subtype === BBReferenceFieldSubType.USER &&
constraints?.type !== "array"
if (isDeprecatedSingleUser && Array.isArray(row[key])) {
row[key] = row[key][0]
if (isDeprecatedSingleUser && Array.isArray(row.get(key))) {
row.set(key, row.get(key)[0])
}
}
await row.save()
return [
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
this.buildRowObject(
sheet.headerValues,
row.toObject(),
row.rowNumber
),
]
} else {
throw new Error("Row does not exist.")

21
packages/server/src/utilities/rowProcessor/attachments.ts
View File

@ -1,6 +1,12 @@
import { ObjectStoreBuckets } from "../../constants"
import { context, db as dbCore, objectStore } from "@budibase/backend-core"
import { FieldType, RenameColumn, Row, Table } from "@budibase/types"
import {
FieldType,
RenameColumn,
Row,
RowAttachment,
Table,
} from "@budibase/types"
export class AttachmentCleanup {
static async coreCleanup(fileListFn: () => string[]): Promise<void> {
@ -21,7 +27,7 @@ export class AttachmentCleanup {
private static extractAttachmentKeys(
type: FieldType,
rowData: any
rowData: RowAttachment[] | RowAttachment
): string[] {
if (
type !== FieldType.ATTACHMENTS &&
@ -35,10 +41,15 @@ export class AttachmentCleanup {
return []
}
if (type === FieldType.ATTACHMENTS) {
return rowData.map((attachment: any) => attachment.key)
if (type === FieldType.ATTACHMENTS && Array.isArray(rowData)) {
return rowData
.filter(attachment => attachment.key)
.map(attachment => attachment.key)
} else if ("key" in rowData) {
return [rowData.key]
}
return [rowData.key]
return []
}
private static async tableChange(

40
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -1,11 +1,11 @@
import * as linkRows from "../../db/linkedRows"
import { processFormulas, fixAutoColumnSubType } from "./utils"
import { fixAutoColumnSubType, processFormulas } from "./utils"
import { objectStore, utils } from "@budibase/backend-core"
import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map"
import {
FieldType,
AutoFieldSubType,
FieldType,
Row,
RowAttachment,
Table,
@ -224,30 +224,32 @@ export async function outputProcessing<T extends Row[] | Row>(
opts.squash = true
}
// process complex types: attachements, bb references...
// process complex types: attachments, bb references...
for (let [property, column] of Object.entries(table.schema)) {
if (column.type === FieldType.ATTACHMENTS) {
for (let row of enriched) {
if (row[property] == null || !Array.isArray(row[property])) {
continue
}
row[property].forEach((attachment: RowAttachment) => {
if (!attachment.url) {
attachment.url = objectStore.getAppFileUrl(attachment.key)
}
})
}
} else if (
if (
column.type === FieldType.ATTACHMENTS ||
column.type === FieldType.ATTACHMENT_SINGLE ||
column.type === FieldType.SIGNATURE
) {
for (let row of enriched) {
if (!row[property] || Object.keys(row[property]).length === 0) {
if (row[property] == null) {
continue
}
if (!row[property].url) {
row[property].url = objectStore.getAppFileUrl(row[property].key)
const process = (attachment: RowAttachment) => {
if (!attachment.url && attachment.key) {
attachment.url = objectStore.getAppFileUrl(attachment.key)
}
return attachment
}
if (typeof row[property] === "string") {
row[property] = JSON.parse(row[property])
}
if (Array.isArray(row[property])) {
row[property].forEach((attachment: RowAttachment) => {
process(attachment)
})
} else {
process(row[property])
}
}
} else if (

83
yarn.lock
View File

@ -5272,11 +5272,6 @@
resolved "https://registry.yarnpkg.com/@types/global-agent/-/global-agent-2.1.1.tgz#3f93185e48a3a36e377a52a8301320cd162a831b"
integrity sha512-sVox8Phk1UKgP6LQPAdeRxfww6vHKt7Bf59dXzYLsQBUEMEn8S10a+ESp/yO0i4fJ3WS4+CIuz42hgJcuA+3mA==
"@types/google-spreadsheet@3.1.5":
version "3.1.5"
resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-3.1.5.tgz#2bdc6f9f5372551e0506cb6ef3f562adcf44fc2e"
integrity sha512-7N+mDtZ1pmya2RRFPPl4KYc2TRgiqCNBLUZfyrKfER+u751JgCO+C24/LzF70UmUm/zhHUbzRZ5mtfaxekQ1ZQ==
"@types/graceful-fs@^4.1.3":
version "4.1.6"
resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.6.tgz#e14b2576a1c25026b7f02ede1de3b84c3a1efeae"
@ -6983,7 +6978,7 @@ axios-retry@^3.1.9:
"@babel/runtime" "^7.15.4"
is-retry-allowed "^2.2.0"
axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.21.4, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.5.0:
axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0:
version "1.6.3"
resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@ -11071,17 +11066,6 @@ gauge@^4.0.3:
strip-ansi "^6.0.1"
wide-align "^1.1.5"
gaxios@^4.0.0:
version "4.3.3"
resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.3.tgz#d44bdefe52d34b6435cc41214fdb160b64abfc22"
integrity sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA==
dependencies:
abort-controller "^3.0.0"
extend "^3.0.2"
https-proxy-agent "^5.0.0"
is-stream "^2.0.0"
node-fetch "^2.6.7"
gaxios@^5.0.0, gaxios@^5.0.1:
version "5.1.3"
resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-5.1.3.tgz#f7fa92da0fe197c846441e5ead2573d4979e9013"
@ -11092,14 +11076,6 @@ gaxios@^5.0.0, gaxios@^5.0.1:
is-stream "^2.0.0"
node-fetch "^2.6.9"
gcp-metadata@^4.2.0:
version "4.3.1"
resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9"
integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==
dependencies:
gaxios "^4.0.0"
json-bigint "^1.0.0"
gcp-metadata@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-5.3.0.tgz#6f45eb473d0cb47d15001476b48b663744d25408"
@ -11506,36 +11482,6 @@ gonzales-pe@^4.2.3, gonzales-pe@^4.3.0:
dependencies:
minimist "^1.2.5"
google-auth-library@7.12.0:
version "7.12.0"
resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.12.0.tgz#7965db6bc20cb31f2df05a08a296bbed6af69426"
integrity sha512-RS/whvFPMoF1hQNxnoVET3DWKPBt1Xgqe2rY0k+Jn7TNhoHlwdnSe7Rlcbo2Nub3Mt2lUVz26X65aDQrWp6x8w==
dependencies:
arrify "^2.0.0"
base64-js "^1.3.0"
ecdsa-sig-formatter "^1.0.11"
fast-text-encoding "^1.0.0"
gaxios "^4.0.0"
gcp-metadata "^4.2.0"
gtoken "^5.0.4"
jws "^4.0.0"
lru-cache "^6.0.0"
google-auth-library@^6.1.3:
version "6.1.6"
resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.6.tgz#deacdcdb883d9ed6bac78bb5d79a078877fdf572"
integrity sha512-Q+ZjUEvLQj/lrVHF/IQwRo6p3s8Nc44Zk/DALsN+ac3T4HY/g/3rrufkgtl+nZ1TW7DNAw5cTChdVp4apUXVgQ==
dependencies:
arrify "^2.0.0"
base64-js "^1.3.0"
ecdsa-sig-formatter "^1.0.11"
fast-text-encoding "^1.0.0"
gaxios "^4.0.0"
gcp-metadata "^4.2.0"
gtoken "^5.0.4"
jws "^4.0.0"
lru-cache "^6.0.0"
google-auth-library@^8.0.1, google-auth-library@^8.0.2:
version "8.9.0"
resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-8.9.0.tgz#15a271eb2ec35d43b81deb72211bd61b1ef14dd0"
@ -11572,13 +11518,6 @@ google-gax@^3.5.7:
protobufjs-cli "1.1.1"
retry-request "^5.0.0"
google-p12-pem@^3.1.3:
version "3.1.4"
resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.4.tgz#123f7b40da204de4ed1fbf2fd5be12c047fc8b3b"
integrity sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg==
dependencies:
node-forge "^1.3.1"
google-p12-pem@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-4.0.1.tgz#82841798253c65b7dc2a4e5fe9df141db670172a"
@ -11586,13 +11525,12 @@ google-p12-pem@^4.0.0:
dependencies:
node-forge "^1.3.1"
google-spreadsheet@3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.2.0.tgz#ce8aa75c15705aa950ad52b091a6fc4d33dcb329"
integrity sha512-z7XMaqb+26rdo8p51r5O03u8aPLAPzn5YhOXYJPcf2hdMVr0dUbIARgdkRdmGiBeoV/QoU/7VNhq1MMCLZv3kQ==
google-spreadsheet@4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#92e30fdba7e0d78c55d50731528df7835d58bfee"
integrity sha512-HFBweDAkOcyC2qO9kmWESKbNuOcn+R7UzZN/tj5LLNxVv8FHmg113u0Ow+yaKwwIOt/NnDtPLuptAhaxTs0FYw==
dependencies:
axios "^0.21.4"
google-auth-library "^6.1.3"
axios "^1.4.0"
lodash "^4.17.21"
gopd@^1.0.1:
@ -11678,15 +11616,6 @@ graphemer@^1.4.0:
resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6"
integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==
gtoken@^5.0.4:
version "5.3.2"
resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.2.tgz#deb7dc876abe002178e0515e383382ea9446d58f"
integrity sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ==
dependencies:
gaxios "^4.0.0"
google-p12-pem "^3.1.3"
jws "^4.0.0"
gtoken@^6.1.0:
version "6.1.2"
resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-6.1.2.tgz#aeb7bdb019ff4c3ba3ac100bbe7b6e74dce0e8bc"