MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge

This commit is contained in:
Martin McKeaveney 2024-09-20 17:39:19 +01:00
parent 876d837c33 67b5d54d37
commit 95a08514b4
23 changed files with 223 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/budibase_ci.yml vendored
View File

@ -108,7 +108,7 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.3.3 &
docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
docker pull redis &
wait $(jobs -p)
@ -179,7 +179,7 @@ jobs:
docker pull minio/minio &
docker pull redis &
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb:v3.3.3 &
docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 &
wait $(jobs -p)

2
charts/budibase/values.yaml
View File

@ -641,7 +641,7 @@ couchdb:
# @ignore
repository: budibase/couchdb
# @ignore
tag: v3.3.3
tag: v3.3.3-sqs-v2.1.1
# @ignore
pullPolicy: Always

2
globalSetup.ts
View File

@ -46,7 +46,7 @@ export default async function setup() {
await killContainers(containers)
try {
const couchdb = new GenericContainer("budibase/couchdb:v3.3.3")
const couchdb = new GenericContainer("budibase/couchdb:v3.3.3-sqs-v2.1.1")
.withExposedPorts(5984, 4984)
.withEnvironment({
COUCHDB_PASSWORD: "budibase",

2
hosting/single/Dockerfile
View File

@ -1,4 +1,4 @@
ARG BASEIMG=budibase/couchdb:v3.3.3
ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
FROM node:20-slim as build
# install node-gyp dependencies

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.5",
"version": "2.32.6",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/backend-core/src/environment.ts
View File

@ -143,6 +143,7 @@ const environment = {
POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,

7
packages/backend-core/src/features/index.ts
View File

@ -6,7 +6,12 @@ import tracer from "dd-trace"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,

1
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -148,6 +148,7 @@ describe("feature flags", () => {
const env: Partial<typeof environment> = {
TENANT_FEATURE_FLAGS: environmentFlags,
SELF_HOSTED: false,
POSTHOG_FEATURE_FLAGS_ENABLED: "true",
}
if (posthogFlags) {

4
packages/backend-core/tests/core/utilities/mocks/licenses.ts
View File

@ -102,10 +102,6 @@ export const useAppBuilders = () => {
return useFeature(Feature.APP_BUILDERS)
}
export const useViewReadonlyColumns = () => {
return useFeature(Feature.VIEW_READONLY_COLUMNS)
}
// QUOTAS
export const setAutomationLogsQuota = (value: number) => {

3
packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
View File

@ -1,6 +1,6 @@
<script>
import { viewsV2 } from "stores/builder"
import { admin, licensing } from "stores/portal"
import { admin } from "stores/portal"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@ -30,7 +30,6 @@
showAvatars={false}
on:updatedatasource={handleGridViewUpdate}
isCloud={$admin.cloud}
allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
>
<svelte:fragment slot="filter">

5
packages/builder/src/stores/portal/licensing.js
View File

@ -140,10 +140,6 @@ export const createLicensingStore = () => {
Constants.Features.VIEW_PERMISSIONS
)
const isViewReadonlyColumnsEnabled = license.features.includes(
Constants.Features.VIEW_READONLY_COLUMNS
)
const budibaseAIEnabled = license.features.includes(
Constants.Features.BUDIBASE_AI
)
@ -173,7 +169,6 @@ export const createLicensingStore = () => {
triggerAutomationRunEnabled,
isViewPermissionsEnabled,
perAppBuildersEnabled,
isViewReadonlyColumnsEnabled,
}
})
},

4
packages/client/src/utils/conditions.js
View File

@ -1,4 +1,5 @@
import { QueryUtils } from "@budibase/frontend-core"
import { EmptyFilterOption } from "@budibase/types"
export const getActiveConditions = conditions => {
if (!conditions?.length) {
@ -33,7 +34,8 @@ export const getActiveConditions = conditions => {
value: condition.referenceValue,
}
const query = QueryUtils.buildQuery([luceneCondition])
let query = QueryUtils.buildQuery([luceneCondition])
query.onEmptyFilter = EmptyFilterOption.RETURN_NONE
const result = QueryUtils.runQuery([luceneCondition], query)
return result.length > 0
})

14
packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
View File

@ -4,16 +4,13 @@
import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
import { FieldPermissions } from "../../../constants"
export let allowViewReadonlyColumns = false
const { columns, datasource } = getContext("grid")
let open = false
let anchor
$: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
$: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
$: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns"
$: permissions =
$datasource.type === "viewV2"
? [
@ -22,9 +19,6 @@
FieldPermissions.HIDDEN,
]
: [FieldPermissions.WRITABLE, FieldPermissions.HIDDEN]
$: disabledPermissions = allowViewReadonlyColumns
? []
: [FieldPermissions.READONLY]
</script>
<div bind:this={anchor}>
@ -41,9 +35,5 @@
</div>
<Popover bind:open {anchor} align="left">
<ColumnsSettingContent
columns={$columns}
{permissions}
{disabledPermissions}
/>
<ColumnsSettingContent columns={$columns} {permissions} />
</Popover>

4
packages/frontend-core/src/components/grid/layout/Grid.svelte
View File

@ -58,7 +58,6 @@
export let buttons = null
export let darkMode
export let isCloud = null
export let allowViewReadonlyColumns = false
export let rowConditions = null
// Unique identifier for DOM nodes inside this instance
@ -115,7 +114,6 @@
buttons,
darkMode,
isCloud,
allowViewReadonlyColumns,
rowConditions,
})
@ -157,7 +155,7 @@
<div class="controls-left">
<slot name="filter" />
<SortButton />
<ColumnsSettingButton {allowViewReadonlyColumns} />
<ColumnsSettingButton />
<SizeButton />
<slot name="controls" />
</div>

2
packages/server/package.json
View File

@ -80,7 +80,7 @@
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3",
"google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
"ioredis": "5.3.2",
"isolated-vm": "^4.7.2",
"jimp": "0.22.12",

2
packages/server/src/api/controllers/row/index.ts
View File

@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
const { tableId } = utils.getSourceId(ctx)
const processedRows = request.rows.map(row => {
let processedRow: Row = typeof row == "string" ? { _id: row } : row
let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row
return !processedRow._rev
? addRev(fixRow(processedRow, ctx.params), tableId)
: fixRow(processedRow, ctx.params)

12
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -1138,6 +1138,18 @@ describe.each([
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to delete a row with ID only", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const createdRow2 = await config.api.row.save(table._id!, {})

84
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -309,10 +309,6 @@ describe.each([
})
describe("readonly fields", () => {
beforeEach(() => {
mocks.licenses.useViewReadonlyColumns()
})
it("readonly fields are persisted", async () => {
const table = await config.api.table.save(
saveTableRequest({
@ -436,7 +432,7 @@ describe.each([
})
})
it("readonly fields cannot be used on free license", async () => {
it("readonly fields can be used on free license", async () => {
mocks.licenses.useCloudFree()
const table = await config.api.table.save(
saveTableRequest({
@ -466,11 +462,7 @@ describe.each([
}
await config.api.viewV2.create(newView, {
status: 400,
body: {
message: "Readonly fields are not enabled",
status: 400,
},
status: 201,
})
})
})
@ -513,7 +505,6 @@ describe.each([
})
it("display fields can be readonly", async () => {
mocks.licenses.useViewReadonlyColumns()
const table = await config.api.table.save(
saveTableRequest({
schema: {
@ -588,7 +579,6 @@ describe.each([
})
it("can update all fields", async () => {
mocks.licenses.useViewReadonlyColumns()
const tableId = table._id!
const updatedData: Required<UpdateViewRequest> = {
@ -802,71 +792,6 @@ describe.each([
)
})
it("cannot update views with readonly on on free license", async () => {
mocks.licenses.useViewReadonlyColumns()
view = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: true,
},
},
})
mocks.licenses.useCloudFree()
await config.api.viewV2.update(view, {
status: 400,
body: {
message: "Readonly fields are not enabled",
},
})
})
it("can remove readonly config after license downgrade", async () => {
mocks.licenses.useViewReadonlyColumns()
view = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: true,
},
Category: {
visible: true,
readonly: true,
},
},
})
mocks.licenses.useCloudFree()
const res = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: false,
},
},
})
expect(res).toEqual(
expect.objectContaining({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: false,
},
},
})
)
})
isInternal &&
it("updating schema will only validate modified field", async () => {
let view = await config.api.viewV2.create({
@ -1046,7 +971,6 @@ describe.each([
})
it("should be able to fetch readonly config after downgrades", async () => {
mocks.licenses.useViewReadonlyColumns()
const res = await config.api.viewV2.create({
name: generator.name(),
tableId: table._id!,
@ -1112,8 +1036,6 @@ describe.each([
})
it("rejects if field is readonly in any view", async () => {
mocks.licenses.useViewReadonlyColumns()
await config.api.viewV2.create({
name: "view a",
tableId: table._id!,
@ -1538,7 +1460,6 @@ describe.each([
})
it("can't persist readonly columns", async () => {
mocks.licenses.useViewReadonlyColumns()
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
@ -1607,7 +1528,6 @@ describe.each([
})
it("can't update readonly columns", async () => {
mocks.licenses.useViewReadonlyColumns()
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),

43
packages/server/src/integrations/googlesheets.ts
View File

@ -330,15 +330,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
return { tables: {}, errors: {} }
}
await this.connect()
const sheets = this.client.sheetsByIndex
const tables: Record<string, Table> = {}
let errors: Record<string, string> = {}
await utils.parallelForeach(
sheets,
async sheet => {
// must fetch rows to determine schema
try {
await sheet.getRows()
await sheet.getRows({ limit: 1 })
} catch (err) {
// We expect this to always be an Error so if it's not, rethrow it to
// make sure we don't fail quietly.
@ -346,26 +347,34 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
throw err
}
if (err.message.startsWith("No values in the header row")) {
errors[sheet.title] = err.message
} else {
// If we get an error we don't expect, rethrow to avoid failing
// quietly.
throw err
if (
err.message.startsWith("No values in the header row") ||
err.message.startsWith("All your header cells are blank")
) {
errors[
sheet.title
] = `Failed to find a header row in sheet "${sheet.title}", is the first row blank?`
return
}
return
}
const id = buildExternalTableId(datasourceId, sheet.title)
tables[sheet.title] = this.getTableSchema(
sheet.title,
sheet.headerValues,
datasourceId,
id
)
// If we get an error we don't expect, rethrow to avoid failing
// quietly.
throw err
}
},
10
)
for (const sheet of sheets) {
const id = buildExternalTableId(datasourceId, sheet.title)
tables[sheet.title] = this.getTableSchema(
sheet.title,
sheet.headerValues,
datasourceId,
id
)
}
let externalTables = finaliseExternalTables(tables, entities)
errors = { ...errors, ...checkExternalTables(externalTables) }
return { tables: externalTables, errors }

107
packages/server/src/integrations/tests/googlesheets.spec.ts
View File

@ -244,6 +244,20 @@ describe("Google Sheets Integration", () => {
expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`))
)
})
it("can export rows", async () => {
const resp = await config.api.row.exportRows(table._id!, {})
const parsed = JSON.parse(resp)
expect(parsed.length).toEqual(2)
expect(parsed[0]).toMatchObject({
name: "Test Contact 1",
description: "original description 1",
})
expect(parsed[1]).toMatchObject({
name: "Test Contact 2",
description: "original description 2",
})
})
})
describe("update", () => {
@ -491,4 +505,97 @@ describe("Google Sheets Integration", () => {
expect(emptyRows.length).toEqual(0)
})
})
describe("fetch schema", () => {
it("should fail to import a completely blank sheet", async () => {
mock.createSheet({ title: "Sheet1" })
await config.api.datasource.fetchSchema(
{
datasourceId: datasource._id!,
tablesFilter: ["Sheet1"],
},
{
status: 200,
body: {
errors: {
Sheet1:
'Failed to find a header row in sheet "Sheet1", is the first row blank?',
},
},
}
)
})
it("should fail to import multiple sheets with blank headers", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2"],
},
{
status: 200,
body: {
errors: {
Sheet1:
'Failed to find a header row in sheet "Sheet1", is the first row blank?',
Sheet2:
'Failed to find a header row in sheet "Sheet2", is the first row blank?',
},
},
}
)
})
it("should only fail the sheet with missing headers", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
mock.createSheet({ title: "Sheet3" })
mock.set("Sheet1!A1", "name")
mock.set("Sheet1!B1", "dob")
mock.set("Sheet2!A1", "name")
mock.set("Sheet2!B1", "dob")
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2", "Sheet3"],
},
{
status: 200,
body: {
errors: {
Sheet3:
'Failed to find a header row in sheet "Sheet3", is the first row blank?',
},
},
}
)
})
it("should only succeed if sheet with missing headers is not being imported", async () => {
mock.createSheet({ title: "Sheet1" })
mock.createSheet({ title: "Sheet2" })
mock.createSheet({ title: "Sheet3" })
mock.set("Sheet1!A1", "name")
mock.set("Sheet1!B1", "dob")
mock.set("Sheet2!A1", "name")
mock.set("Sheet2!B1", "dob")
await config.api.datasource.fetchSchema(
{
datasourceId: datasource!._id!,
tablesFilter: ["Sheet1", "Sheet2"],
},
{
status: 200,
body: { errors: {} },
}
)
})
})
})

82
packages/server/src/integrations/tests/utils/googlesheets.ts
View File

@ -22,6 +22,7 @@ import type {
CellPadding,
Color,
GridRange,
DataSourceSheetProperties,
} from "google-spreadsheet/src/lib/types/sheets-types"
const BLACK: Color = { red: 0, green: 0, blue: 0 }
@ -91,7 +92,7 @@ interface UpdateValuesResponse {
// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest
interface AddSheetRequest {
properties: WorksheetProperties
properties: Partial<WorksheetProperties>
}
// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse
@ -236,6 +237,38 @@ export class GoogleSheetsMock {
this.mockAPI()
}
public cell(cell: string): Value | undefined {
const cellData = this.cellData(cell)
if (!cellData) {
return undefined
}
return this.cellValue(cellData)
}
public set(cell: string, value: Value): void {
const cellData = this.cellData(cell)
if (!cellData) {
throw new Error(`Cell ${cell} not found`)
}
cellData.userEnteredValue = this.createValue(value)
}
public sheet(name: string | number): Sheet | undefined {
if (typeof name === "number") {
return this.getSheetById(name)
}
return this.getSheetByName(name)
}
public createSheet(opts: Partial<WorksheetProperties>): Sheet {
const properties = this.defaultWorksheetProperties(opts)
if (this.getSheetByName(properties.title)) {
throw new Error(`Sheet ${properties.title} already exists`)
}
const resp = this.handleAddSheet({ properties })
return this.getSheetById(resp.properties.sheetId)!
}
private route(
method: "get" | "put" | "post",
path: string | RegExp,
@ -462,35 +495,39 @@ export class GoogleSheetsMock {
return response
}
private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
const properties: Omit<WorksheetProperties, "dataSourceSheetProperties"> = {
private defaultWorksheetProperties(
opts: Partial<WorksheetProperties>
): WorksheetProperties {
return {
index: this.spreadsheet.sheets.length,
hidden: false,
rightToLeft: false,
tabColor: BLACK,
tabColorStyle: { rgbColor: BLACK },
sheetType: "GRID",
title: request.properties.title,
title: "Sheet",
sheetId: this.spreadsheet.sheets.length,
gridProperties: {
rowCount: 100,
columnCount: 26,
frozenRowCount: 0,
frozenColumnCount: 0,
hideGridlines: false,
rowGroupControlAfter: false,
columnGroupControlAfter: false,
},
dataSourceSheetProperties: {} as DataSourceSheetProperties,
...opts,
}
}
private handleAddSheet(request: AddSheetRequest): AddSheetResponse {
const properties = this.defaultWorksheetProperties(request.properties)
this.spreadsheet.sheets.push({
properties: properties as WorksheetProperties,
data: [this.createEmptyGrid(100, 26)],
properties,
data: [
this.createEmptyGrid(
properties.gridProperties.rowCount,
properties.gridProperties.columnCount
),
],
})
// dataSourceSheetProperties is only returned by the API if the sheet type is
// DATA_SOURCE, which we aren't using, so sadly we need to cast here.
return { properties: properties as WorksheetProperties }
return { properties }
}
private handleDeleteRange(request: DeleteRangeRequest) {
@ -767,21 +804,6 @@ export class GoogleSheetsMock {
return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex)
}
public cell(cell: string): Value | undefined {
const cellData = this.cellData(cell)
if (!cellData) {
return undefined
}
return this.cellValue(cellData)
}
public sheet(name: string | number): Sheet | undefined {
if (typeof name === "number") {
return this.getSheetById(name)
}
return this.getSheetByName(name)
}
private getCellNumericIndexes(
sheet: Sheet | number,
row: number,

9
packages/server/src/sdk/app/views/index.ts
View File

@ -5,13 +5,11 @@ import {
Table,
TableSchema,
View,
ViewFieldMetadata,
ViewV2,
ViewV2ColumnEnriched,
ViewV2Enriched,
} from "@budibase/types"
import { HTTPError } from "@budibase/backend-core"
import { features } from "@budibase/pro"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
@ -59,13 +57,6 @@ async function guardViewSchema(
}
if (viewSchema[field].readonly) {
if (
!(await features.isViewReadonlyColumnsEnabled()) &&
!(tableSchemaField as ViewFieldMetadata).readonly
) {
throw new HTTPError(`Readonly fields are not enabled`, 400)
}
if (!viewSchema[field].visible) {
throw new HTTPError(
`Field "${field}" must be visible if you want to make it readonly`,

2
scripts/build-single-image-sqs.sh
View File

@ -2,4 +2,4 @@
yarn build:apps
version=$(./scripts/getCurrentVersion.sh)
docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs .
docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 .