From 015ef56110a7c8b06bdc86001d20cd17cb929e07 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 09:29:05 +0100 Subject: [PATCH 001/195] wip --- packages/backend-core/src/sql/sql.ts | 50 +++++++- .../server/src/api/controllers/row/views.ts | 14 ++- .../src/api/controllers/view/viewsV2.ts | 51 ++++++-- .../src/api/routes/tests/viewV2.spec.ts | 71 +++++++++++- packages/server/src/sdk/app/rows/search.ts | 3 + .../src/sdk/app/rows/search/internal/sqs.ts | 23 +++- packages/server/src/sdk/app/views/index.ts | 109 ++++++++++++++---- packages/shared-core/src/helpers/index.ts | 1 + packages/shared-core/src/helpers/schema.ts | 4 + packages/shared-core/src/helpers/views.ts | 39 +++++++ packages/types/src/api/web/app/rows.ts | 1 + packages/types/src/documents/app/view.ts | 18 +-- packages/types/src/sdk/row.ts | 9 +- packages/types/src/sdk/search.ts | 2 + packages/types/src/shared/typeUtils.ts | 23 ++++ 15 files changed, 358 insertions(+), 60 deletions(-) create mode 100644 packages/shared-core/src/helpers/views.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e6738d4b36..b2fec5fe57 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -10,10 +10,12 @@ import { } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { + Aggregation, AnySearchFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, + CalculationType, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -789,6 +791,38 @@ class InternalBuilder { return query.countDistinct(`${aliased}.${primary[0]} as total`) } + addAggregations( + query: Knex.QueryBuilder, + aggregations: Aggregation[] + ): Knex.QueryBuilder { + const fields = this.query.resource?.fields || [] + if (fields.length > 0) { + query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + } + for (const aggregation of aggregations) { + const op = aggregation.calculationType + const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + switch (op) { + case CalculationType.COUNT: + query = query.count(field) + break + case CalculationType.SUM: + query = query.sum(field) + break + case CalculationType.AVG: + query = query.avg(field) + break + case CalculationType.MIN: + query = query.min(field) + break + case CalculationType.MAX: + query = query.max(field) + break + } + } + return query + } + addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort } = this.query const primaryKey = this.table.primary @@ -1172,10 +1206,18 @@ class InternalBuilder { } } - // if counting, use distinct count, else select - query = !counting - ? query.select(this.generateSelectStatement()) - : this.addDistinctCount(query) + const aggregations = this.query.resource?.aggregations || [] + if (counting) { + query = this.addDistinctCount(query) + } else if (aggregations.length > 0) { + query = query.select( + this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") + ) + query = this.addAggregations(query, aggregations) + } else { + query = query.select(this.generateSelectStatement()) + } + // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { query = this.addSorting(query) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index d2541dfa25..ba48bc5664 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,8 +7,9 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + Aggregation, } from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" import sdk from "../../../sdk" import { db, context, features } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" @@ -27,7 +28,7 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } - const viewFields = Object.entries(view.schema || {}) + const viewFields = Object.entries(helpers.views.basicFields(view)) .filter(([_, value]) => value.visible) .map(([key]) => key) const { body } = ctx.request @@ -74,6 +75,14 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) + const aggregations: Aggregation[] = Object.entries( + helpers.views.calculationFields(view) + ).map(([name, { field, calculationType }]) => ({ + name, + calculationType, + field, + })) + const searchOptions: RequiredKeys & RequiredKeys> = { tableId: view.tableId, @@ -84,6 +93,7 @@ export async function searchView( bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, + aggregations, } const result = await sdk.rows.search(searchOptions) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 4208772fa6..18306ba245 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -8,8 +8,45 @@ import { ViewResponse, ViewResponseEnriched, ViewV2, + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" +import { helpers } from "@budibase/shared-core" + +function stripUnknownFields( + field: ViewUIFieldMetadata +): RequiredKeys { + if (helpers.views.isCalculationField(field)) { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + calculationType: field.calculationType, + field: field.field, + } + return strippedField + } else { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + } + return strippedField + } +} + +function stripUndefinedFields(obj: Record): void { + Object.keys(obj) + .filter(key => obj[key] === undefined) + .forEach(key => { + delete obj[key] + }) +} async function parseSchema(view: CreateViewRequest) { if (!view.schema) { @@ -18,18 +55,8 @@ async function parseSchema(view: CreateViewRequest) { const finalViewSchema = view.schema && Object.entries(view.schema).reduce((p, [fieldName, schemaValue]) => { - const fieldSchema: RequiredKeys = { - order: schemaValue.order, - width: schemaValue.width, - visible: schemaValue.visible, - readonly: schemaValue.readonly, - icon: schemaValue.icon, - } - Object.entries(fieldSchema) - .filter(([, val]) => val === undefined) - .forEach(([key]) => { - delete fieldSchema[key as keyof ViewUIFieldMetadata] - }) + const fieldSchema = stripUnknownFields(schemaValue) + stripUndefinedFields(fieldSchema) p[fieldName] = fieldSchema return p }, {} as Record>) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 356f01dee0..72f55c16d2 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -18,6 +18,7 @@ import { ViewV2, SearchResponse, BasicOperator, + CalculationType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -32,13 +33,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1978,6 +1979,64 @@ describe.each([ }) ) }) + + describe("calculations", () => { + let table: Table + let rows: Row[] + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it.only("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) + ) + }) + }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 3d5de2e6cb..95cbc919a1 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -46,6 +46,9 @@ export async function search( paginate: options.paginate, fields: options.fields, countRows: options.countRows, + aggregations: options.aggregations + ?.map(a => `${a.field}:${a.calculationType}`) + .join(", "), }) const isExternalTable = isExternalTableID(options.tableId) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index fb140e3c14..93c6cab2ea 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -48,6 +48,7 @@ import { } from "@budibase/shared-core" import { isSearchingByRowID } from "../utils" import tracer from "dd-trace" +import { cloneDeep } from "lodash" const builder = new sql.Sql(SqlClient.SQL_LITE) const SQLITE_COLUMN_LIMIT = 2000 @@ -285,7 +286,7 @@ export async function search( table: Table, opts?: { retrying?: boolean } ): Promise> { - let { paginate, query, ...params } = options + let { paginate, query, ...params } = cloneDeep(options) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) @@ -303,6 +304,21 @@ export async function search( ...cleanupFilters(query, table, allTables), documentType: DocumentType.ROW, } + + let fields = options.fields + if (fields === undefined) { + fields = buildInternalFieldList(table, allTables, { relationships }) + } else { + fields = fields.map(f => mapToUserColumn(f)) + } + + if (options.aggregations) { + options.aggregations = options.aggregations.map(a => { + a.field = mapToUserColumn(a.field) + return a + }) + } + const request: QueryJson = { endpoint: { // not important, we query ourselves @@ -317,9 +333,7 @@ export async function search( tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, - resource: { - fields: buildInternalFieldList(table, allTables, { relationships }), - }, + resource: { fields, aggregations: options.aggregations }, relationships, } @@ -426,6 +440,7 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } + throw err throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index ed624c2b5c..01c2d3d2fa 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,8 +1,9 @@ import { + BasicViewUIFieldMetadata, RenameColumn, + Table, TableSchema, View, - ViewUIFieldMetadata, ViewV2, ViewV2Enriched, } from "@budibase/types" @@ -38,31 +39,84 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +async function guardCalculationViewSchema( + table: Table, + view: Omit +) { + const calculationFields = helpers.views.calculationFields(view) + for (const calculationFieldName of Object.keys(calculationFields)) { + const schema = calculationFields[calculationFieldName] + const targetSchema = table.schema[schema.field] + if (!targetSchema) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`, + 400 + ) + } + + if (!helpers.schema.isNumeric(targetSchema)) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`, + 400 + ) + } + } + + const groupByFields = helpers.views.basicFields(view) + for (const groupByFieldName of Object.keys(groupByFields)) { + const targetSchema = table.schema[groupByFieldName] + if (!targetSchema) { + throw new HTTPError( + `Group by field "${groupByFieldName}" does not exist in the table schema`, + 400 + ) + } + } +} + async function guardViewSchema( tableId: string, view: Omit ) { - const viewSchema = view.schema || {} const table = await sdk.tables.getTable(tableId) + if (helpers.views.isCalculationView(view)) { + await guardCalculationViewSchema(table, view) + } + + await checkReadonlyFields(table, view) + checkRequiredFields(table, view) + checkDisplayField(view) +} + +async function checkReadonlyFields( + table: Table, + view: Omit +) { + const viewSchema = view.schema || {} for (const field of Object.keys(viewSchema)) { - const tableSchemaField = table.schema[field] - if (!tableSchemaField) { + const viewFieldSchema = viewSchema[field] + if (helpers.views.isCalculationField(viewFieldSchema)) { + continue + } + + const tableFieldSchema = table.schema[field] + if (!tableFieldSchema) { throw new HTTPError( `Field "${field}" is not valid for the requested table`, 400 ) } - if (viewSchema[field].readonly) { + if (viewFieldSchema.readonly) { if ( !(await features.isViewReadonlyColumnsEnabled()) && - !(tableSchemaField as ViewUIFieldMetadata).readonly + !(tableFieldSchema as BasicViewUIFieldMetadata).readonly ) { throw new HTTPError(`Readonly fields are not enabled`, 400) } - if (!viewSchema[field].visible) { + if (!viewFieldSchema.visible) { throw new HTTPError( `Field "${field}" must be visible if you want to make it readonly`, 400 @@ -70,18 +124,33 @@ async function guardViewSchema( } } } +} - const existingView = - table?.views && (table.views[view.name] as ViewV2 | undefined) +function checkDisplayField(view: Omit) { + if (view.primaryDisplay) { + const viewSchemaField = view.schema?.[view.primaryDisplay] + if (!viewSchemaField?.visible) { + throw new HTTPError( + `You can't hide "${view.primaryDisplay}" because it is the display column.`, + 400 + ) + } + } +} + +function checkRequiredFields( + table: Table, + view: Omit +) { + const existingView = table.views?.[view.name] as ViewV2 | undefined for (const field of Object.values(table.schema)) { if (!helpers.schema.isRequired(field.constraints)) { continue } - const viewSchemaField = viewSchema[field.name] - const existingViewSchema = - existingView?.schema && existingView.schema[field.name] + const viewSchemaField = view.schema?.[field.name] + const existingViewSchema = existingView?.schema?.[field.name] if (!viewSchemaField && !existingViewSchema?.visible) { // Supporting existing configs with required columns but hidden in views continue @@ -94,24 +163,16 @@ async function guardViewSchema( ) } - if (viewSchemaField.readonly) { + if ( + helpers.views.isBasicViewField(viewSchemaField) && + viewSchemaField.readonly + ) { throw new HTTPError( `You can't make "${field.name}" readonly because it is a required field.`, 400 ) } } - - if (view.primaryDisplay) { - const viewSchemaField = viewSchema[view.primaryDisplay] - - if (!viewSchemaField?.visible) { - throw new HTTPError( - `You can't hide "${view.primaryDisplay}" because it is the display column.`, - 400 - ) - } - } } export async function create( diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts index c33ff2ab1e..503f71e4eb 100644 --- a/packages/shared-core/src/helpers/index.ts +++ b/packages/shared-core/src/helpers/index.ts @@ -2,3 +2,4 @@ export * from "./helpers" export * from "./integrations" export * as cron from "./cron" export * as schema from "./schema" +export * as views from "./views" diff --git a/packages/shared-core/src/helpers/schema.ts b/packages/shared-core/src/helpers/schema.ts index d0035cc305..c9200fee18 100644 --- a/packages/shared-core/src/helpers/schema.ts +++ b/packages/shared-core/src/helpers/schema.ts @@ -45,3 +45,7 @@ export function decodeNonAscii(str: string): string { String.fromCharCode(parseInt(p1, 16)) ) } + +export function isNumeric(field: FieldSchema) { + return field.type === FieldType.NUMBER || field.type === FieldType.BIGINT +} diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts new file mode 100644 index 0000000000..0364ccff41 --- /dev/null +++ b/packages/shared-core/src/helpers/views.ts @@ -0,0 +1,39 @@ +import { + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, + ViewUIFieldMetadata, + ViewV2, +} from "@budibase/types" +import { pickBy } from "lodash" + +export function isCalculationField( + field: ViewUIFieldMetadata +): field is ViewCalculationFieldMetadata { + return "calculationType" in field +} + +export function isBasicViewField( + field: ViewUIFieldMetadata +): field is BasicViewUIFieldMetadata { + return !isCalculationField(field) +} + +type UnsavedViewV2 = Omit + +export function isCalculationView(view: UnsavedViewV2) { + return Object.values(view.schema || {}).some(isCalculationField) +} + +export function calculationFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, isCalculationField) +} + +export function basicFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, field => !isCalculationField(field)) +} diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index ce6f6f672d..5d49ac1812 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -26,6 +26,7 @@ export interface SearchViewRowRequest | "paginate" | "query" | "countRows" + | "aggregations" > {} export interface SearchRowResponse { diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 24dad0bcca..539c1e0d23 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -33,10 +33,19 @@ export interface View { groupBy?: string } -export type ViewUIFieldMetadata = UIFieldMetadata & { +export interface BasicViewUIFieldMetadata extends UIFieldMetadata { readonly?: boolean } +export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata { + calculationType: CalculationType + field: string +} + +export type ViewUIFieldMetadata = + | BasicViewUIFieldMetadata + | ViewCalculationFieldMetadata + export enum CalculationType { SUM = "sum", AVG = "avg", @@ -45,11 +54,6 @@ export enum CalculationType { MAX = "max", } -export type ViewCalculationFieldMetadata = ViewUIFieldMetadata & { - calculationType: CalculationType - field: string -} - export interface ViewV2 { version: 2 id: string @@ -62,7 +66,7 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: Record } export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 6850359cc3..8ee0338731 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -1,8 +1,14 @@ import { SortOrder, SortType } from "../api" import { SearchFilters } from "./search" -import { Row } from "../documents" +import { CalculationType, Row } from "../documents" import { WithRequired } from "../shared" +export interface Aggregation { + name: string + calculationType: CalculationType + field: string +} + export interface SearchParams { tableId?: string query?: SearchFilters @@ -18,6 +24,7 @@ export interface SearchParams { indexer?: () => Promise rows?: Row[] countRows?: boolean + aggregations?: Aggregation[] } // when searching for rows we want a more extensive search type that requires certain properties diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..ba3c388480 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -2,6 +2,7 @@ import { Operation } from "./datasources" import { Row, Table, DocumentType } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" +import { Aggregation } from "./row" export enum BasicOperator { EQUAL = "equal", @@ -143,6 +144,7 @@ export interface QueryJson { } resource?: { fields: string[] + aggregations?: Aggregation[] } filters?: SearchFilters sort?: SortJson diff --git a/packages/types/src/shared/typeUtils.ts b/packages/types/src/shared/typeUtils.ts index c7ecebed0a..dbb3fc2553 100644 --- a/packages/types/src/shared/typeUtils.ts +++ b/packages/types/src/shared/typeUtils.ts @@ -4,6 +4,29 @@ export type DeepPartial = { export type ISO8601 = string +/** + * RequiredKeys make it such that you _must_ assign a value to every key in the + * type. It differs subtly from Required in that it doesn't change the type + * of the fields, you can specify undefined as a value and that's fine. + * + * Example: + * + * ```ts + * interface Foo { + * bar: string + * baz?: string + * } + * + * type FooRequiredKeys = RequiredKeys + * type FooRequired = Required + * + * const a: FooRequiredKeys = { bar: "hello", baz: undefined } + * const b: FooRequired = { bar: "hello", baz: undefined } + * ``` + * + * In this code, a passes type checking whereas b does not. This is because + * Required makes baz non-optional. + */ export type RequiredKeys = { [K in keyof Required]: T[K] } From 594551e1b550368609f1e998f356e2dd28b354f3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 14:05:32 +0100 Subject: [PATCH 002/195] tests for filter steps --- .../tests/scenarios/scenarios.spec.ts | 167 +++++++++++++++++- .../tests/utilities/AutomationTestBuilder.ts | 9 + 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 40d6094525..62a1b9db8f 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -1,8 +1,9 @@ import * as automation from "../../index" import * as setup from "../utilities" -import { LoopStepType, FieldType } from "@budibase/types" +import { LoopStepType, FieldType, Table } from "@budibase/types" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" import { DatabaseName } from "../../../integrations/tests/utils" +import { FilterConditions } from "../../../automations/steps/filter" describe("Automation Scenarios", () => { let config = setup.getConfig() @@ -196,4 +197,168 @@ describe("Automation Scenarios", () => { ) }) }) + describe.only("Automations with filter", () => { + let table: Table + + beforeEach(async () => { + table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + }) + + it("should stop an automation if the condition is not met", async () => { + const builder = createAutomationBuilder({ + name: "Test Equal", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: "Equal Test", + value: 10, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition: FilterConditions.EQUAL, + value: 20, + }) + .serverLog({ text: "Equal condition met" }) + .run() + + expect(results.steps[2].outputs.success).toBeTrue() + expect(results.steps[2].outputs.result).toBeFalse() + expect(results.steps[3]).toBeUndefined() + }) + + it("should continue the automation if the condition is met", async () => { + const builder = createAutomationBuilder({ + name: "Test Not Equal", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: "Not Equal Test", + value: 10, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition: FilterConditions.NOT_EQUAL, + value: 20, + }) + .serverLog({ text: "Not Equal condition met" }) + .run() + + expect(results.steps[2].outputs.success).toBeTrue() + expect(results.steps[2].outputs.result).toBeTrue() + expect(results.steps[3].outputs.success).toBeTrue() + }) + + const testCases = [ + { + condition: FilterConditions.EQUAL, + value: 10, + rowValue: 10, + expectPass: true, + }, + { + condition: FilterConditions.NOT_EQUAL, + value: 10, + rowValue: 20, + expectPass: true, + }, + { + condition: FilterConditions.GREATER_THAN, + value: 10, + rowValue: 15, + expectPass: true, + }, + { + condition: FilterConditions.LESS_THAN, + value: 10, + rowValue: 5, + expectPass: true, + }, + { + condition: FilterConditions.GREATER_THAN, + value: 10, + rowValue: 5, + expectPass: false, + }, + { + condition: FilterConditions.LESS_THAN, + value: 10, + rowValue: 15, + expectPass: false, + }, + ] + + testCases.forEach(({ condition, value, rowValue, expectPass }) => { + it(`should ${ + expectPass ? "pass" : "fail" + } the filter when condition is "${condition}" and value is ${value}`, async () => { + const builder = createAutomationBuilder({ + name: `Test ${condition}`, + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow({ + row: { + name: `${condition} Test`, + value: rowValue, + tableId: table._id, + }, + }) + .queryRows({ + tableId: table._id!, + }) + .filter({ + field: "{{ steps.2.rows.0.value }}", + condition, + value, + }) + .serverLog({ + text: `${condition} condition ${expectPass ? "passed" : "failed"}`, + }) + .run() + + expect(results.steps[2].outputs.result).toBe(expectPass) + if (expectPass) { + expect(results.steps[3].outputs.success).toBeTrue() + } else { + expect(results.steps[3]).toBeUndefined() + } + }) + }) + }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f477efabe4..6c89bd1f60 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -33,6 +33,7 @@ import { BranchStepInputs, SearchFilters, Branch, + FilterStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -161,6 +162,14 @@ class BaseStepBuilder { input ) } + + filter(input: FilterStepInputs): this { + return this.step( + AutomationActionStepId.FILTER, + BUILTIN_ACTION_DEFINITIONS.FILTER, + input + ) + } } class StepBuilder extends BaseStepBuilder { build(): AutomationStep[] { From c85bc88bf9099d2e4f9082e5bb2baff00fedeaa9 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Sep 2024 11:54:04 +0100 Subject: [PATCH 003/195] draft of openai impl --- packages/backend-core/src/configs/configs.ts | 9 ++++ .../server/src/automations/steps/openai.ts | 51 ++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..00ccee8c4e 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,6 @@ import { + AIConfig, + AIInnerConfig, Config, ConfigType, GoogleConfig, @@ -254,3 +256,10 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + +// AI +export async function getAIConfig(): Promise { + const config = await getConfig(ConfigType.AI) + return config?.config +} + diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 1c148b2e73..d8017d0ceb 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -10,6 +10,7 @@ import { } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" +import * as pro from "@budibase/pro" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", @@ -60,6 +61,23 @@ export const definition: AutomationStepDefinition = { }, } +async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { + const openai = new OpenAI({ + apiKey: env.OPENAI_API_KEY, + }) + + const completion = await openai.chat.completions.create({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt, + }, + ], + }) + return completion?.choices[0]?.message?.content +} + export async function run({ inputs, }: { @@ -81,20 +99,27 @@ export async function run({ } try { - const openai = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }) + let response + const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() + const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const completion = await openai.chat.completions.create({ - model: inputs.model, - messages: [ - { - role: "user", - content: inputs.prompt, - }, - ], - }) - const response = completion?.choices[0]?.message?.content + if (budibaseAIEnabled || customConfigsEnabled) { + // Enterprise has custom configs + // if custom configs are enabled full stop + // Don't use their budibase AI credits, unless it uses the budibase AI configuration + // TODO: grab the config from the database (maybe wrap this in the pro AI module) + // TODO: pass it into the model to execute the prompt + + // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config + // Make sure it uses their credits + // Should be handled in the LLM wrapper in pro + const llm = new pro.ai.LLMWrapper() + await llm.init() + response = await llm.run(inputs.prompt) + } else { + // fallback to the default that uses the environment variable for backwards compat + response = await legacyOpenAIPrompt(inputs) + } return { response, From 224be10b6c21e7d69c9c1d3dc63cabca75236f0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 16:08:59 +0100 Subject: [PATCH 004/195] support for overriding models --- packages/backend-core/src/configs/configs.ts | 6 +- packages/pro | 2 +- packages/server/package.json | 2 +- .../server/src/automations/steps/openai.ts | 2 +- .../src/api/controllers/global/configs.ts | 34 +-- yarn.lock | 223 ++++++++++++++---- 6 files changed, 189 insertions(+), 80 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 00ccee8c4e..379c1de7a4 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -258,8 +258,8 @@ export async function getSCIMConfig(): Promise { } // AI -export async function getAIConfig(): Promise { - const config = await getConfig(ConfigType.AI) - return config?.config + +export async function getAIConfig(): Promise { + return getConfig(ConfigType.AI) } diff --git a/packages/pro b/packages/pro index 922431260e..ff141defc6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 922431260e90d558a1ca55398475412e75088057 +Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f diff --git a/packages/server/package.json b/packages/server/package.json index 6dfd528963..41ce10c135 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -101,7 +101,7 @@ "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", - "openai": "^4.52.1", + "openai": "4.59.0", "openapi-types": "9.3.1", "oracledb": "6.5.1", "pg": "8.10.0", diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 1dffc37d1f..13b0008654 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -115,7 +115,7 @@ export async function run({ // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config // Make sure it uses their credits // Should be handled in the LLM wrapper in pro - const llm = new pro.ai.LLMWrapper() + const llm = new pro.ai.LargeLanguageModel(inputs.model) await llm.init() response = await llm.run(inputs.prompt) } else { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 70b2279f6c..d7a18713a2 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -334,32 +334,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { ) } -async function enrichAIConfig(aiConfig: AIConfig) { - // Strip out the API Keys from the response so they don't show in the UI - for (const key in aiConfig.config) { - if (aiConfig.config[key].apiKey) { - aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT - } - } - - // Return the Budibase AI data source as part of the response if licensing allows - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const defaultConfigExists = Object.keys(aiConfig.config).some( - key => aiConfig.config[key].isDefault - ) - if (budibaseAIEnabled) { - aiConfig.config["budibase_ai"] = { - provider: "OpenAI", - active: true, - isDefault: !defaultConfigExists, - defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "", - name: "Budibase AI", - } - } - - return aiConfig -} - export async function find(ctx: UserCtx) { try { // Find the config with the most granular scope based on context @@ -372,7 +346,13 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await enrichAIConfig(scopedConfig) + await pro.ai.getAIConfig(scopedConfig) + // Strip out the API Keys from the response so they don't show in the UI + for (const key in scopedConfig.config) { + if (scopedConfig.config[key].apiKey) { + scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT + } + } } ctx.body = scopedConfig } else { diff --git a/yarn.lock b/yarn.lock index 110cbd7a15..69c3978aaf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,6 +33,19 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.24" +"@anthropic-ai/sdk@^0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475" + integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.1.2" resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8" @@ -759,20 +772,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@4.2.1", "@azure/identity@^3.4.1": - version "4.2.1" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" - integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== +"@azure/identity@^3.4.1": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" + integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.3.0" + "@azure/core-util" "^1.6.1" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.11.1" - "@azure/msal-node" "^2.9.2" + "@azure/msal-browser" "^3.5.0" + "@azure/msal-node" "^2.5.1" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -803,7 +816,7 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.11.1": +"@azure/msal-browser@^3.5.0": version "3.23.0" resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== @@ -815,7 +828,7 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== -"@azure/msal-node@^2.9.2": +"@azure/msal-node@^2.5.1": version "2.13.1" resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== @@ -6040,6 +6053,11 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== +"@types/qs@^6.9.15": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -7448,7 +7466,30 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11290,7 +11331,14 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5: + version "4.2.5" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" + integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== + dependencies: + strnum "^1.0.5" + +fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11544,6 +11592,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0, follow-redirects@^1.14.4: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12637,7 +12690,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13162,7 +13220,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@~1.1.6: +is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -13648,11 +13706,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14562,7 +14615,14 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@^3.0.2, kind-of@^3.1.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -16205,7 +16265,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16399,13 +16459,25 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17020,19 +17092,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^4.52.1: - version "4.52.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" - integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== +openai@4.59.0: + version "4.59.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08" + integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" + "@types/qs" "^6.9.15" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" - web-streams-polyfill "^3.2.1" + qs "^6.10.3" openapi-response-validator@^9.2.0: version "9.3.1" @@ -17556,7 +17629,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18862,7 +18943,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19934,6 +20015,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20006,13 +20092,33 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21575,7 +21681,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21585,6 +21691,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22113,14 +22227,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22522,11 +22628,6 @@ web-streams-polyfill@4.0.0-beta.3: resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== -web-streams-polyfill@^3.2.1: - version "3.3.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" - integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== - web-vitals@^4.0.1: version "4.2.3" resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7" @@ -22900,10 +23001,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22913,6 +23037,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From e40f397c25a96e9cdab5f8ad134637a850c72a0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 16:29:44 +0100 Subject: [PATCH 005/195] move check for OpenAI config variable to legacy path --- packages/pro | 2 +- .../server/src/automations/steps/openai.ts | 30 ++++++++----------- .../src/api/controllers/global/configs.ts | 2 +- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/pro b/packages/pro index ff141defc6..72e9846146 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f +Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 13b0008654..79b6664674 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -63,6 +63,11 @@ export const definition: AutomationStepDefinition = { }, } +/** + * Maintains backward compatibility with automation steps created before the introduction + * of custom configurations and Budibase AI + * @param inputs - automation inputs from the OpenAI automation step. + */ async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY, @@ -85,14 +90,6 @@ export async function run({ }: { inputs: OpenAIStepInputs }): Promise { - if (!env.OPENAI_API_KEY) { - return { - success: false, - response: - "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", - } - } - if (inputs.prompt == null) { return { success: false, @@ -106,20 +103,19 @@ export async function run({ const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() if (budibaseAIEnabled || customConfigsEnabled) { - // Enterprise has custom configs - // if custom configs are enabled full stop - // Don't use their budibase AI credits, unless it uses the budibase AI configuration - // TODO: grab the config from the database (maybe wrap this in the pro AI module) - // TODO: pass it into the model to execute the prompt - - // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config - // Make sure it uses their credits - // Should be handled in the LLM wrapper in pro const llm = new pro.ai.LargeLanguageModel(inputs.model) await llm.init() response = await llm.run(inputs.prompt) } else { // fallback to the default that uses the environment variable for backwards compat + if (!env.OPENAI_API_KEY) { + return { + success: false, + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + response = await legacyOpenAIPrompt(inputs) } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index d7a18713a2..f53868d7df 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await pro.ai.getAIConfig(scopedConfig) + await pro.ai.enrichAIConfig(scopedConfig) // Strip out the API Keys from the response so they don't show in the UI for (const key in scopedConfig.config) { if (scopedConfig.config[key].apiKey) { From 8e2d21c85fb85d38a217422d914d3bf86efd6589 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 18:27:49 +0100 Subject: [PATCH 006/195] call right sdk method on ai module for enriching AI config --- packages/pro | 2 +- packages/worker/src/api/controllers/global/configs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 72e9846146..69633aa64b 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3 +Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f53868d7df..f01c7c8869 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await pro.ai.enrichAIConfig(scopedConfig) + await pro.sdk.ai.enrichAIConfig(scopedConfig) // Strip out the API Keys from the response so they don't show in the UI for (const key in scopedConfig.config) { if (scopedConfig.config[key].apiKey) { From d7334f100b263e358492e7f79456dd2f0ca72462 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 01:21:13 +0000 Subject: [PATCH 007/195] Bump next from 14.1.1 to 14.2.10 in /examples/nextjs-api-sales Bumps [next](https://github.com/vercel/next.js) from 14.1.1 to 14.2.10. - [Release notes](https://github.com/vercel/next.js/releases) - [Changelog](https://github.com/vercel/next.js/blob/canary/release.js) - [Commits](https://github.com/vercel/next.js/compare/v14.1.1...v14.2.10) --- updated-dependencies: - dependency-name: next dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- examples/nextjs-api-sales/package.json | 2 +- examples/nextjs-api-sales/yarn.lock | 124 +++++++++++++------------ 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 7ecf264add..f1ef4843a1 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "14.1.1", + "next": "14.2.10", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index a44956ba21..9acbdfdeb6 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" - integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== +"@next/env@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c" + integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,50 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" - integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== +"@next/swc-darwin-arm64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80" + integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ== -"@next/swc-darwin-x64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" - integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== +"@next/swc-darwin-x64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8" + integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA== -"@next/swc-linux-arm64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" - integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== +"@next/swc-linux-arm64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20" + integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA== -"@next/swc-linux-arm64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" - integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== +"@next/swc-linux-arm64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47" + integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ== -"@next/swc-linux-x64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" - integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== +"@next/swc-linux-x64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082" + integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg== -"@next/swc-linux-x64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" - integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== +"@next/swc-linux-x64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3" + integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA== -"@next/swc-win32-arm64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" - integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== +"@next/swc-win32-arm64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209" + integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ== -"@next/swc-win32-ia32-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" - integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== +"@next/swc-win32-ia32-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041" + integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg== -"@next/swc-win32-x64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" - integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== +"@next/swc-win32-x64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455" + integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -129,11 +129,17 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@types/json5@^0.0.29": @@ -1245,28 +1251,28 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" - integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== +next@14.2.10: + version "14.2.10" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6" + integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww== dependencies: - "@next/env" "14.1.1" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.10" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.1" - "@next/swc-darwin-x64" "14.1.1" - "@next/swc-linux-arm64-gnu" "14.1.1" - "@next/swc-linux-arm64-musl" "14.1.1" - "@next/swc-linux-x64-gnu" "14.1.1" - "@next/swc-linux-x64-musl" "14.1.1" - "@next/swc-win32-arm64-msvc" "14.1.1" - "@next/swc-win32-ia32-msvc" "14.1.1" - "@next/swc-win32-x64-msvc" "14.1.1" + "@next/swc-darwin-arm64" "14.2.10" + "@next/swc-darwin-x64" "14.2.10" + "@next/swc-linux-arm64-gnu" "14.2.10" + "@next/swc-linux-arm64-musl" "14.2.10" + "@next/swc-linux-x64-gnu" "14.2.10" + "@next/swc-linux-x64-musl" "14.2.10" + "@next/swc-win32-arm64-msvc" "14.2.10" + "@next/swc-win32-ia32-msvc" "14.2.10" + "@next/swc-win32-x64-msvc" "14.2.10" node-domexception@^1.0.0: version "1.0.0" From 2cf07a40f6467aaac25416509c1e6f528bb42c95 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 09:44:33 +0100 Subject: [PATCH 008/195] fix for setting default configs --- .../src/pages/builder/portal/settings/ai/index.svelte | 5 ++++- packages/pro | 2 +- packages/server/src/automations/steps/openai.ts | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 2ac1609e7c..b60ea24dbc 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -56,10 +56,13 @@ } else { // We don't store the default BB AI config in the DB delete fullAIConfig.config.budibase_ai + // unset the default value from other configs if default is set if (editingAIConfig.isDefault) { for (let key in fullAIConfig.config) { - fullAIConfig.config[key].isDefault = false + if (key !== id) { + fullAIConfig.config[key].isDefault = false + } } } // Add new or update existing custom AI Config diff --git a/packages/pro b/packages/pro index 69633aa64b..5daf17e325 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c +Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 79b6664674..5eff12db47 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -115,7 +115,6 @@ export async function run({ "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", } } - response = await legacyOpenAIPrompt(inputs) } From cb8d0984b1fa484e4482dd7a2009842c71d293d0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 10:52:52 +0100 Subject: [PATCH 009/195] update Providers --- packages/types/src/documents/global/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 8d64b49ee9..33f7e10584 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,7 +111,7 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom" +export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" export interface AIInnerConfig { [key: string]: { From d4db493519bfe45e662ad5c3c4bdfd5e3b4be6bc Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 18 Sep 2024 11:50:15 +0100 Subject: [PATCH 010/195] Set view permissions to explicit roles from the parent table --- .../server/src/api/controllers/permission.ts | 4 +-- packages/server/src/sdk/app/views/index.ts | 33 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 66a3254348..b75af88067 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -20,7 +20,7 @@ import { import { removeFromArray } from "../../utilities" import sdk from "../../sdk" -const enum PermissionUpdateType { +export const enum PermissionUpdateType { REMOVE = "remove", ADD = "add", } @@ -37,7 +37,7 @@ async function getAllDBRoles(db: Database) { return body.rows.map(row => row.doc!) } -async function updatePermissionOnRole( +export async function updatePermissionOnRole( { roleId, resourceId, diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index d7e05abf2f..c580bfde50 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,5 +1,6 @@ import { FieldType, + PermissionLevel, RelationSchemaField, RenameColumn, Table, @@ -10,20 +11,22 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { HTTPError, roles } from "@budibase/backend-core" import { features } from "@budibase/pro" import { helpers, PROTECTED_EXTERNAL_COLUMNS, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" - import * as utils from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" - import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" +import { + updatePermissionOnRole, + PermissionUpdateType, +} from "src/api/controllers/permission" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -123,8 +126,30 @@ export async function create( viewRequest: Omit ): Promise { await guardViewSchema(tableId, viewRequest) + const view = await pickApi(tableId).create(tableId, viewRequest) - return pickApi(tableId).create(tableId, viewRequest) + // Set permissions to be the same as the table + const tablePerms = await sdk.permissions.getResourcePerms(tableId) + const readRole = tablePerms[PermissionLevel.READ]?.role + const writeRole = tablePerms[PermissionLevel.WRITE]?.role + await updatePermissionOnRole( + { + roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.READ, + }, + PermissionUpdateType.ADD + ) + await updatePermissionOnRole( + { + roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.WRITE, + }, + PermissionUpdateType.ADD + ) + + return view } export async function update(tableId: string, view: ViewV2): Promise { From 9cbaa02f5f305542453493d0e9520cf0ca7681bc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 14:01:54 +0100 Subject: [PATCH 011/195] add test for automation run step when LLM feature flags are on --- packages/pro | 2 +- .../src/automations/tests/openai.spec.ts | 35 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 5daf17e325..320f8ecf8a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4 +Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11 diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3a5a57475a..0a64411a12 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -4,6 +4,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import * as pro from "@budibase/pro" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -23,6 +24,20 @@ jest.mock("openai", () => ({ })), })) +jest.mock("@budibase/pro", () => ({ + ...jest.requireActual("@budibase/pro"), + ai: { + LargeLanguageModel: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, + features: { + isAICustomConfigsEnabled: jest.fn(), + isBudibaseAIEnabled: jest.fn(), + } +})) + const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -41,6 +56,7 @@ describe("test the openai action", () => { afterEach(() => { resetEnv() + jest.clearAllMocks() }) afterAll(_afterAll) @@ -86,7 +102,7 @@ describe("test the openai action", () => { ) const res = await runStep("OPENAI", { - prompt: OPENAI_PROMPT, + prompt: OPENAI_PROMPT,jj }) expect(res.response).toEqual( @@ -94,4 +110,21 @@ describe("test the openai action", () => { ) expect(res.success).toBeFalsy() }) + + it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true) + jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) + + const prompt = "What is the meaning of life?" + await runStep("OPENAI", { + model: "gpt-4o-mini", + prompt + }) + + expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + + const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value + expect(llmInstance.init).toHaveBeenCalled() + expect(llmInstance.run).toHaveBeenCalledWith(prompt) + }) }) From 69b2bf9bdb80aad9350ecb6ebf4b4f82555a7d5f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 14:21:10 +0100 Subject: [PATCH 012/195] lint --- packages/backend-core/src/configs/configs.ts | 1 - packages/server/src/automations/tests/openai.spec.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 379c1de7a4..6c2f2fe586 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -262,4 +262,3 @@ export async function getSCIMConfig(): Promise { export async function getAIConfig(): Promise { return getConfig(ConfigType.AI) } - diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 0a64411a12..498d15b93c 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -35,7 +35,7 @@ jest.mock("@budibase/pro", () => ({ features: { isAICustomConfigsEnabled: jest.fn(), isBudibaseAIEnabled: jest.fn(), - } + }, })) const mockedOpenAI = OpenAI as jest.MockedClass @@ -102,7 +102,7 @@ describe("test the openai action", () => { ) const res = await runStep("OPENAI", { - prompt: OPENAI_PROMPT,jj + prompt: OPENAI_PROMPT, }) expect(res.response).toEqual( @@ -118,7 +118,7 @@ describe("test the openai action", () => { const prompt = "What is the meaning of life?" await runStep("OPENAI", { model: "gpt-4o-mini", - prompt + prompt, }) expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") From 8a399bfbdfd33a8b117ee08ad0612befeb05f24c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 17:42:55 +0100 Subject: [PATCH 013/195] set custom configs quotas to right number --- packages/pro | 2 +- packages/worker/src/api/controllers/global/configs.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 320f8ecf8a..4300b75bde 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11 +Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71 diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f01c7c8869..f3262dc0dd 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,6 +253,8 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } + const numConfigs = Object.keys(config).length + await pro.quotas.updateCustomAIConfigCount(numConfigs) break } } catch (err: any) { From 950bd45b2c1abd58a0bc46f1248dd7cdac76b2c0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 18:04:13 +0100 Subject: [PATCH 014/195] add ts ignore for mock --- packages/server/src/automations/tests/openai.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 498d15b93c..342288a6a1 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -123,6 +123,7 @@ describe("test the openai action", () => { expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + // @ts-ignore const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) From e0fb44f63fbde2997d1c1c4765409ccf6f7fc3a2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:21:49 +0100 Subject: [PATCH 015/195] lint --- packages/backend-core/src/configs/configs.ts | 1 - .../src/api/controllers/global/configs.ts | 3 +- yarn.lock | 266 ++++++++---------- 3 files changed, 115 insertions(+), 155 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 6c2f2fe586..e4f4a874a5 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,6 +1,5 @@ import { AIConfig, - AIInnerConfig, Config, ConfigType, GoogleConfig, diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f3262dc0dd..e6e80ff3a5 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,8 +253,7 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } - const numConfigs = Object.keys(config).length - await pro.quotas.updateCustomAIConfigCount(numConfigs) + await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length) break } } catch (err: any) { diff --git a/yarn.lock b/yarn.lock index 69c3978aaf..84d64637e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -772,20 +772,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@^3.4.1": - version "3.4.2" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" - integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== +"@azure/identity@4.2.1", "@azure/identity@^3.4.1": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" + integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.6.1" + "@azure/core-util" "^1.3.0" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.5.0" - "@azure/msal-node" "^2.5.1" + "@azure/msal-browser" "^3.11.1" + "@azure/msal-node" "^2.9.2" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -816,7 +816,7 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.5.0": +"@azure/msal-browser@^3.11.1": version "3.23.0" resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== @@ -828,7 +828,7 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== -"@azure/msal-node@^2.5.1": +"@azure/msal-node@^2.9.2": version "2.13.1" resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== @@ -2066,6 +2066,44 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/backend-core@2.32.5": + version "0.0.0" + dependencies: + "@budibase/nano" "10.1.5" + "@budibase/pouchdb-replication-stream" "1.2.11" + "@budibase/shared-core" "0.0.0" + "@budibase/types" "0.0.0" + aws-cloudfront-sign "3.0.2" + aws-sdk "2.1030.0" + bcrypt "5.1.0" + bcryptjs "2.4.3" + bull "4.10.1" + correlation-id "4.0.0" + dd-trace "5.2.0" + dotenv "16.0.1" + ioredis "5.3.2" + joi "17.6.0" + jsonwebtoken "9.0.2" + knex "2.4.2" + koa-passport "^6.0.0" + koa-pino-logger "4.0.0" + lodash "4.17.21" + node-fetch "2.6.7" + passport-google-oauth "2.0.0" + passport-local "1.0.0" + passport-oauth2-refresh "^2.1.0" + pino "8.11.0" + pino-http "8.3.3" + posthog-node "4.0.1" + pouchdb "7.3.0" + pouchdb-find "7.2.2" + redlock "4.2.0" + rotating-file-stream "3.1.0" + sanitize-s3-objectkey "0.0.1" + semver "^7.5.4" + tar-fs "2.1.1" + uuid "^8.3.2" + "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2108,6 +2146,45 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "2.32.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea" + integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA== + dependencies: + "@budibase/backend-core" "2.32.5" + "@budibase/shared-core" "2.32.5" + "@budibase/string-templates" "2.32.5" + "@budibase/types" "2.32.5" + "@koa/router" "8.0.8" + bull "4.10.1" + dd-trace "5.2.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + +"@budibase/shared-core@2.32.5": + version "0.0.0" + dependencies: + "@budibase/types" "0.0.0" + cron-validate "1.4.5" + +"@budibase/string-templates@2.32.5": + version "0.0.0" + dependencies: + "@budibase/handlebars-helpers" "^0.13.2" + dayjs "^1.10.8" + handlebars "^4.7.8" + lodash.clonedeep "^4.5.0" + +"@budibase/types@2.32.5": + version "0.0.0" + dependencies: + scim-patch "^0.8.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7466,30 +7543,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0: - version "0.24.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" - integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== - dependencies: - follow-redirects "^1.14.4" - -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11331,14 +11385,7 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5: - version "4.2.5" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" - integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== - dependencies: - strnum "^1.0.5" - -fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11592,11 +11639,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.14.4: - version "1.15.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" - integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== - follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12690,12 +12732,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13220,7 +13257,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -13706,6 +13743,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14615,14 +14657,7 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@^3.0.2, kind-of@^3.1.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -16265,7 +16300,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16459,25 +16494,13 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9, node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17629,15 +17652,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18943,7 +18958,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -20015,11 +20030,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20092,33 +20102,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21681,7 +21671,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21691,14 +21681,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22227,6 +22209,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -23001,33 +22991,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -23037,11 +23004,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 062732b29d3514010c1b17743d74c0e673d414a8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:36:00 +0100 Subject: [PATCH 016/195] lint --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 4300b75bde..5929fbb3c6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71 +Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e From 6a5f001977a4cfeda3f9d1842d2716165262df4a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:54:25 +0100 Subject: [PATCH 017/195] remove tests that are no longer relevant in worker --- .../controllers/global/tests/configs.spec.ts | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 3ff6a5298c..ad2e2aa974 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -35,55 +35,6 @@ describe("Global configs controller", () => { }) }) - it("Should return the default BB AI config when the feature is turned on", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(true)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - budibase_ai: { - provider: "OpenAI", - active: true, - isDefault: true, - name: "Budibase AI", - defaultModel: "", - }, - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - - it("Should not not return the default Budibase AI config when on self host", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(false)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - it("Should not update existing secrets when updating an existing AI Config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data) From 55c7751dbbc6f96f66cc1b8d35d7561f797b528c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 19 Sep 2024 15:12:03 +0100 Subject: [PATCH 018/195] Move permission updates into SDK --- .../server/src/api/controllers/permission.ts | 107 ++---------------- .../server/src/sdk/app/permissions/index.ts | 102 ++++++++++++++++- packages/server/src/sdk/app/views/index.ts | 5 +- 3 files changed, 111 insertions(+), 103 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index b75af88067..55b942686f 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -3,7 +3,6 @@ import { UserCtx, Database, Role, - PermissionLevel, GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, @@ -12,107 +11,15 @@ import { RemovePermissionRequest, RemovePermissionResponse, } from "@budibase/types" -import { getRoleParams } from "../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../utilities/security" -import { removeFromArray } from "../../utilities" import sdk from "../../sdk" - -export const enum PermissionUpdateType { - REMOVE = "remove", - ADD = "add", -} +import { PermissionUpdateType } from "../../sdk/app/permissions" const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// utility function to stop this repetition - permissions always stored under roles -async function getAllDBRoles(db: Database) { - const body = await db.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return body.rows.map(row => row.doc!) -} - -export async function updatePermissionOnRole( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - updateType: PermissionUpdateType -) { - const db = context.getAppDB() - const remove = updateType === PermissionUpdateType.REMOVE - const isABuiltin = roles.isBuiltin(roleId) - const dbRoleId = roles.getDBRoleID(roleId) - const dbRoles = await getAllDBRoles(db) - const docUpdates: Role[] = [] - - // the permission is for a built in, make sure it exists - if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { - const builtin = roles.getBuiltinRoles()[roleId] - builtin._id = roles.getDBRoleID(builtin._id!) - dbRoles.push(builtin) - } - - // now try to find any roles which need updated, e.g. removing the - // resource from another role and then adding to the new role - for (let role of dbRoles) { - let updated = false - const rolePermissions: Record = role.permissions - ? role.permissions - : {} - // make sure its an array, also handle migrating - if ( - !rolePermissions[resourceId] || - !Array.isArray(rolePermissions[resourceId]) - ) { - rolePermissions[resourceId] = - typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId] as unknown as PermissionLevel] - : [] - } - // handle the removal/updating the role which has this permission first - // the updating (role._id !== dbRoleId) is required because a resource/level can - // only be permitted in a single role (this reduces hierarchy confusion and simplifies - // the general UI for this, rather than needing to show everywhere it is used) - if ( - (role._id !== dbRoleId || remove) && - rolePermissions[resourceId].indexOf(level) !== -1 - ) { - removeFromArray(rolePermissions[resourceId], level) - updated = true - } - // handle the adding, we're on the correct role, at it to this - if (!remove && role._id === dbRoleId) { - const set = new Set(rolePermissions[resourceId]) - rolePermissions[resourceId] = [...set.add(level)] - updated = true - } - // handle the update, add it to bulk docs to perform at end - if (updated) { - role.permissions = rolePermissions - docUpdates.push(role) - } - } - - const response = await db.bulkDocs(docUpdates) - return response.map(resp => { - const version = docUpdates.find(role => role._id === resp.id)?.version - const _id = roles.getExternalRoleID(resp.id, version) - return { - _id, - rev: resp.rev, - error: resp.error, - reason: resp.reason, - } - }) -} - export function fetchBuiltin(ctx: UserCtx) { ctx.body = Object.values(permissions.getBuiltinPermissions()) } @@ -124,7 +31,7 @@ export function fetchLevels(ctx: UserCtx) { export async function fetch(ctx: UserCtx) { const db = context.getAppDB() - const dbRoles: Role[] = await getAllDBRoles(db) + const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db) let permissions: any = {} // create an object with structure role ID -> resource ID -> level for (let role of dbRoles) { @@ -186,12 +93,18 @@ export async function getDependantResources( export async function addPermission(ctx: UserCtx) { const params: AddPermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.ADD + ) } export async function removePermission( ctx: UserCtx ) { const params: RemovePermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.REMOVE + ) } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index a6e81652ee..5f8882399b 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,22 +1,34 @@ -import { db, roles } from "@budibase/backend-core" +import { db, roles, context } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, + Role, + Database, } from "@budibase/types" -import { extractViewInfoFromID, isViewID } from "../../../db/utils" +import { + extractViewInfoFromID, + isViewID, + getRoleParams, +} from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../../utilities/security" import sdk from "../../../sdk" import { isV2 } from "../views" +import { removeFromArray } from "../../../utilities" type ResourcePermissions = Record< string, { role: string; type: PermissionSource } > +export const enum PermissionUpdateType { + REMOVE = "remove", + ADD = "add", +} + export async function getInheritablePermissions( resourceId: string ): Promise { @@ -100,3 +112,89 @@ export async function getDependantResources( return } + +export async function updatePermissionOnRole( + { + roleId, + resourceId, + level, + }: { roleId: string; resourceId: string; level: PermissionLevel }, + updateType: PermissionUpdateType +) { + const db = context.getAppDB() + const remove = updateType === PermissionUpdateType.REMOVE + const isABuiltin = roles.isBuiltin(roleId) + const dbRoleId = roles.getDBRoleID(roleId) + const dbRoles = await getAllDBRoles(db) + const docUpdates: Role[] = [] + + // the permission is for a built in, make sure it exists + if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { + const builtin = roles.getBuiltinRoles()[roleId] + builtin._id = roles.getDBRoleID(builtin._id!) + dbRoles.push(builtin) + } + + // now try to find any roles which need updated, e.g. removing the + // resource from another role and then adding to the new role + for (let role of dbRoles) { + let updated = false + const rolePermissions: Record = role.permissions + ? role.permissions + : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId] as unknown as PermissionLevel] + : [] + } + // handle the removal/updating the role which has this permission first + // the updating (role._id !== dbRoleId) is required because a resource/level can + // only be permitted in a single role (this reduces hierarchy confusion and simplifies + // the general UI for this, rather than needing to show everywhere it is used) + if ( + (role._id !== dbRoleId || remove) && + rolePermissions[resourceId].indexOf(level) !== -1 + ) { + removeFromArray(rolePermissions[resourceId], level) + updated = true + } + // handle the adding, we're on the correct role, at it to this + if (!remove && role._id === dbRoleId) { + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] + updated = true + } + // handle the update, add it to bulk docs to perform at end + if (updated) { + role.permissions = rolePermissions + docUpdates.push(role) + } + } + + const response = await db.bulkDocs(docUpdates) + return response.map(resp => { + const version = docUpdates.find(role => role._id === resp.id)?.version + const _id = roles.getExternalRoleID(resp.id, version) + return { + _id, + rev: resp.rev, + error: resp.error, + reason: resp.reason, + } + }) +} + +// utility function to stop this repetition - permissions always stored under roles +export async function getAllDBRoles(db: Database) { + const body = await db.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + return body.rows.map(row => row.doc!) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index c580bfde50..47af484ebc 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -23,10 +23,7 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" -import { - updatePermissionOnRole, - PermissionUpdateType, -} from "src/api/controllers/permission" +import { updatePermissionOnRole, PermissionUpdateType } from "../permissions" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { From 418bbff2f57e573ce9f36796f5fb77014aaa4870 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 19 Sep 2024 15:15:19 +0100 Subject: [PATCH 019/195] Lint --- packages/server/src/api/controllers/permission.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 55b942686f..c7afb6a351 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,7 +1,6 @@ import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, - Database, Role, GetResourcePermsResponse, ResourcePermissionInfo, From b45017dd5af5984009e399105a1c6859c5c43d0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 17:06:09 +0100 Subject: [PATCH 020/195] lint --- packages/worker/src/api/controllers/global/tests/configs.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index ad2e2aa974..9091f29247 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,4 +1,3 @@ -import * as pro from "@budibase/pro" import { verifyAIConfig } from "../configs" import { TestConfiguration, structures } from "../../../../tests" import { AIInnerConfig } from "@budibase/types" From af26b915b18e105f07d7a3bf9b525b08a3f246df Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 10:17:45 +0100 Subject: [PATCH 021/195] Add Google Sheets search test. --- .../server/src/integrations/googlesheets.ts | 11 +- .../integrations/tests/googlesheets.spec.ts | 190 ++++++++++++++++++ .../integrations/tests/utils/googlesheets.ts | 2 + 3 files changed, 197 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6012ff7789..831528f84d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rows = await sheet.getRows() } - if (hasFilters && query.paginate) { - rows = rows.slice(offset, offset + limit) - } - const headerValues = sheet.headerValues - let response = rows.map(row => - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber) ) response = dataFilters.runQuery(response, query.filters || {}) + if (hasFilters && query.paginate) { + response = response.slice(offset, offset + limit) + } + if (query.sort) { if (Object.keys(query.sort).length !== 1) { console.warn("Googlesheets does not support multiple sorting", { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index dcf4a61b50..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => { ) }) }) + + describe("search", () => { + let table: Table + + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Foo", + }, + { + name: "Bar", + }, + { + name: "Baz", + }, + ], + }) + }) + + it("should be able to find rows with equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + equal: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with not equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEqual: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should be able to find rows with empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + empty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(0) + }) + + it("should be able to find rows with not empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEmpty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(3) + }) + + it("should be able to find rows with one of filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + oneOf: { + name: ["Foo", "Bar"], + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Foo") + expect(response.rows[1].name).toEqual("Bar") + }) + + it("should be able to find rows with fuzzy filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + fuzzy: { + name: "oo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with range filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + range: { + name: { + low: "A", + high: "C", + }, + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Unique value!`, + })), + }) + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Non-unique value!`, + })), + }) + + let response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + }) + let rows: Row[] = response.rows + + while (response.hasNextPage) { + response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + bookmark: response.bookmark, + }) + + expect(response.rows.length).toBeLessThanOrEqual(10) + rows = rows.concat(response.rows) + } + + // Make sure we only get rows matching the query. + expect(rows.length).toEqual(50) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining( + Array.from({ length: 50 }, () => "Unique value!") + ) + ) + + // Make sure all of the rows have a unique ID. + const ids = Object.keys( + rows.reduce((acc, row) => { + acc[row._id!] = true + return acc + }, {}) + ) + expect(ids.length).toEqual(50) + }) + }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 4747f5f9bf..4b9445ebca 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -440,6 +440,8 @@ export class GoogleSheetsMock { endColumnIndex: 0, }) + sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length + return { spreadsheetId: this.spreadsheet.spreadsheetId, tableRange: range, From fc44b38fc5e5a2cab4a920e4cf0aac6d6d855247 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 11:52:06 +0100 Subject: [PATCH 022/195] Fix tests. --- .../server/src/api/routes/tests/viewV2.spec.ts | 16 ++++++++-------- packages/shared-core/src/helpers/views.ts | 6 ------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 6769c80867..9313e3ab4d 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,13 +37,13 @@ import { import sdk from "../../../sdk" describe.each([ - // ["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2295,7 +2295,7 @@ describe.each([ ) }) - describe("calculations", () => { + describe.skip("calculations", () => { let table: Table let rows: Row[] @@ -2325,7 +2325,7 @@ describe.each([ ) }) - it.only("should be able to search by calculations", async () => { + it("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts index 0364ccff41..c65bc4882d 100644 --- a/packages/shared-core/src/helpers/views.ts +++ b/packages/shared-core/src/helpers/views.ts @@ -25,15 +25,9 @@ export function isCalculationView(view: UnsavedViewV2) { } export function calculationFields(view: UnsavedViewV2) { - if (!isCalculationView(view)) { - throw new Error("View is not a calculation view") - } return pickBy(view.schema || {}, isCalculationField) } export function basicFields(view: UnsavedViewV2) { - if (!isCalculationView(view)) { - throw new Error("View is not a calculation view") - } return pickBy(view.schema || {}, field => !isCalculationField(field)) } From 7d4aa252449dad10f75247125a49b542820cb281 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 20 Sep 2024 11:53:48 +0100 Subject: [PATCH 023/195] fix issue with loop bindings showing for non loop steps --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index af67ae8d22..927b8588b3 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -752,13 +752,21 @@ : allSteps[idx].icon if (wasLoopBlock) { - loopBlockCount++ schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) }) } + + if ( + allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP && + allSteps + .slice(0, blockIdx) + .some(step => step.stepId === ActionStepID.LOOP) + ) { + bindings = bindings.filter(x => !x.readableBinding.includes("loop")) + } return bindings } From ddc360a3da3a1c3c31a5e7b6cb7daebdbd7caaba Mon Sep 17 00:00:00 2001 From: mikesealey Date: Fri, 20 Sep 2024 11:54:26 +0100 Subject: [PATCH 024/195] tweaks sizing of multi-attachment preview in compact setting --- packages/bbui/src/Form/Core/Dropzone.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index c69bf0d6bb..e223d789a1 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -396,6 +396,10 @@ padding: 6px 10px; margin-bottom: 8px; } + + .compact .gallery > * { + max-height: 25px; + } .title { display: flex; flex-direction: row; From 876d837c3301c03eb09fe3cad2692ea3acf47105 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 14:58:10 +0100 Subject: [PATCH 025/195] adjust instantiation of llm class after static initialiser change --- packages/pro | 2 +- packages/server/src/automations/steps/openai.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/pro b/packages/pro index 5929fbb3c6..922431260e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e +Subproject commit 922431260e90d558a1ca55398475412e75088057 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 5eff12db47..b1dfa3df5b 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -103,8 +103,7 @@ export async function run({ const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() if (budibaseAIEnabled || customConfigsEnabled) { - const llm = new pro.ai.LargeLanguageModel(inputs.model) - await llm.init() + const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) response = await llm.run(inputs.prompt) } else { // fallback to the default that uses the environment variable for backwards compat From c5db1d1da305e350804685e59877780aafa45011 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 16:37:23 +0100 Subject: [PATCH 026/195] Got a test passing but I hate it a bit. --- .../src/api/controllers/row/utils/basic.ts | 16 +++++++++++++++- .../src/api/controllers/row/utils/utils.ts | 4 +++- .../server/src/api/routes/tests/viewV2.spec.ts | 16 ++++++++-------- packages/server/src/db/linkedRows/index.ts | 16 ++++++++++++---- packages/server/src/sdk/app/rows/search.ts | 1 + .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++++++--- .../server/src/utilities/rowProcessor/index.ts | 9 +++++++++ 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9c49f5636a..20a1cd0742 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,12 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types" +import { + FieldSchema, + FieldType, + Row, + Table, + JsonTypes, + Aggregation, +} from "@budibase/types" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -84,12 +91,14 @@ export function basicProcessing({ tables, isLinked, sqs, + aggregations, }: { row: Row table: Table tables: Table[] isLinked: boolean sqs?: boolean + aggregations?: Aggregation[] }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) @@ -108,6 +117,11 @@ export function basicProcessing({ thisRow[fieldName] = value } } + + for (let aggregation of aggregations || []) { + thisRow[aggregation.name] = row[aggregation.name] + } + let columns: string[] = Object.keys(table.schema) if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index ac305e70b6..1fc7221294 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -2,6 +2,7 @@ import * as utils from "../../../../db/utils" import { context } from "@budibase/backend-core" import { + Aggregation, Ctx, DatasourcePlusQueryResponse, FieldType, @@ -129,7 +130,7 @@ export async function sqlOutputProcessing( table: Table, tables: Record, relationships: RelationshipsJson[], - opts?: { sqs?: boolean } + opts?: { sqs?: boolean; aggregations?: Aggregation[] } ): Promise { if (isKnexEmptyReadResponse(rows)) { return [] @@ -150,6 +151,7 @@ export async function sqlOutputProcessing( tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, + aggregations: opts?.aggregations, }) if (thisRow._id == null) { throw new Error("Unable to generate row ID for SQL rows") diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aaaf03c113..dc940c5ace 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,13 +37,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2215,7 +2215,7 @@ describe.each([ ) }) - describe.skip("calculations", () => { + describe("calculations", () => { let table: Table let rows: Row[] @@ -2245,7 +2245,7 @@ describe.each([ ) }) - it("should be able to search by calculations", async () => { + it.only("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..4222f9b5e4 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -20,10 +20,11 @@ import { Row, Table, TableSchema, - ViewFieldMetadata, + ViewUIFieldMetadata, ViewV2, } from "@budibase/types" import sdk from "../../sdk" +import { helpers } from "@budibase/shared-core" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -264,12 +265,19 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} - if (options?.fromViewId && allowRelationshipSchemas) { + let viewSchema: Record = {} + if (options?.fromViewId) { const view = Object.values(table.views || {}).find( (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId ) - viewSchema = view?.schema || {} + + if (view && helpers.views.isCalculationView(view)) { + return enriched + } + + if (allowRelationshipSchemas && view) { + viewSchema = view.schema || {} + } } // will populate this as we find them diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 95cbc919a1..dbf0cefd51 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -37,6 +37,7 @@ export async function search( return await tracer.trace("search", async span => { span?.addTags({ tableId: options.tableId, + viewId: options.viewId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 188c95fb5c..b5bf8e752f 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -386,8 +386,9 @@ export async function search( // make sure JSON columns corrected const processed = builder.convertJsonStringColumns( table, - await sqlOutputProcessing(rows, table!, allTablesMap, relationships, { + await sqlOutputProcessing(rows, table, allTablesMap, relationships, { sqs: true, + aggregations: options.aggregations, }) ) @@ -406,11 +407,16 @@ export async function search( preserveLinks: true, squash: true, fromViewId: options.viewId, + aggregations: options.aggregations, }) // check if we need to pick specific rows out if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] + const fields = [ + ...options.fields, + ...PROTECTED_INTERNAL_COLUMNS, + ...(options.aggregations || []).map(a => a.name), + ] finalRows = finalRows.map((r: any) => pick(r, fields)) } @@ -440,6 +446,5 @@ export async function search( return { rows: [] } } throw err - throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f6cf44d6d6..73768fdd57 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,6 +11,7 @@ import { import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { + Aggregation, AutoFieldSubType, FieldType, IdentityType, @@ -250,6 +251,7 @@ export async function outputProcessing( fromRow?: Row skipBBReferences?: boolean fromViewId?: string + aggregations?: Aggregation[] } = { squash: true, preserveLinks: false, @@ -357,6 +359,7 @@ export async function outputProcessing( fromViewId: opts?.fromViewId, }) } + // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) if (isExternal || (await features.flags.isEnabled("SQS"))) { @@ -385,9 +388,15 @@ export async function outputProcessing( const tableFields = Object.keys(table.schema).filter( f => table.schema[f].visible !== false ) + const fields = [...tableFields, ...protectedColumns].map(f => f.toLowerCase() ) + + for (const aggregation of opts.aggregations || []) { + fields.push(aggregation.name.toLowerCase()) + } + for (const row of enriched) { for (const key of Object.keys(row)) { if (!fields.includes(key.toLowerCase())) { From 4975ae7fef1b2bcac717a3706194743d51f0cf8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 16:58:06 +0100 Subject: [PATCH 027/195] Extra test logging. --- .github/workflows/budibase_ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4b9ebf1e5d..388f2000ed 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -186,6 +186,8 @@ jobs: - run: yarn --frozen-lockfile - name: Test server + env: + DEBUG: "testcontainers*" run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} From da805b10ae576f23fd6d8d823d655a3d20382a49 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 17:31:44 +0100 Subject: [PATCH 028/195] Set mssql SHA back to 2019. --- packages/server/datasource-sha.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 61249d530c..9b935ed8eb 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 +MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d From 4fec5fcddd3e49ebdaebfe536ef7685ce3240fbd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 17:48:56 +0100 Subject: [PATCH 029/195] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 922431260e..f088614945 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 922431260e90d558a1ca55398475412e75088057 +Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98 From efdfbe7229b94027077da5d2bc4320f3e01f4c45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 Sep 2024 17:58:31 +0100 Subject: [PATCH 030/195] Fixing an issue that was stopping the limit from being applied to MySQL, it needs to wrap the query the same as all other DBs, however it needs to apply the where statement in a slightly different manner. --- packages/backend-core/src/sql/sql.ts | 55 +++++++++++++++++----------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 55f71d76b0..a9400b1839 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -930,7 +930,8 @@ class InternalBuilder { } const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, - fromAlias = aliases?.[fromTable] || fromTable + fromAlias = aliases?.[fromTable] || fromTable, + throughAlias = (throughTable && aliases?.[throughTable]) || throughTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, @@ -961,30 +962,38 @@ class InternalBuilder { // add sorting to get consistent order .orderBy(primaryKey) - // many-to-many relationship with junction table - if (throughTable && toPrimary && fromPrimary) { - const throughAlias = aliases?.[throughTable] || throughTable + const addCorrelatedWhere = ( + query: Knex.QueryBuilder, + column1: string, + column2: string + ) => { + return query.where( + column1, + "=", + knex.raw(this.quotedIdentifier(column2)) + ) + } + + const isManyToMany = throughTable && toPrimary && fromPrimary + let correlatedTo = isManyToMany + ? `${throughAlias}.${fromKey}` + : `${toAlias}.${toKey}`, + correlatedFrom = isManyToMany + ? `${fromAlias}.${fromPrimary}` + : `${fromAlias}.${fromKey}` + // many-to-many relationship needs junction table join + if (isManyToMany) { let throughTableWithSchema = this.tableNameWithSchema(throughTable, { alias: throughAlias, schema: endpoint.schema, }) - subQuery = subQuery - .join(throughTableWithSchema, function () { - this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) - }) - .where( - `${throughAlias}.${fromKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) - ) + subQuery = subQuery.join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) } - // one-to-many relationship with foreign key - else { - subQuery = subQuery.where( - `${toAlias}.${toKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) - ) + // my-sql needs the where statement to be part of main query, not sub-query + if (sqlClient !== SqlClient.MY_SQL) { + subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) } const standardWrap = (select: string): Knex.QueryBuilder => { @@ -1009,8 +1018,10 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = subQuery.select( - knex.raw(`json_arrayagg(json_object(${fieldList}))`) + wrapperQuery = addCorrelatedWhere( + standardWrap(`json_arrayagg(json_object(${fieldList}))`), + isManyToMany ? fromKey! : toKey!, + correlatedFrom ) break case SqlClient.ORACLE: From 3b984629ac9318899a642a824dfaecf609e999c7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 18:35:34 +0100 Subject: [PATCH 031/195] pro reference --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f088614945..1da5c27fae 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98 +Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea From dc6c6ad9c80e515aeb93508e85a5118fb03c9793 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 18:42:50 +0100 Subject: [PATCH 032/195] ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1da5c27fae..e2fe0f9cc8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea +Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 From 51e09ddf7b2c2d0274cd2b4a503a375993505f1d Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 23 Sep 2024 10:08:21 +0100 Subject: [PATCH 033/195] Update row action tests to revoke explicit view permissions when testing triggering against views --- packages/server/src/api/routes/tests/rowAction.spec.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index ef7d2afbba..efd28eb92f 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -826,11 +826,20 @@ describe("/rowsActions", () => { ) ).id + // Allow row action on view await config.api.rowAction.setViewPermission( tableId, viewId, rowAction.id ) + + // Delete explicit view permissions so they inherit table permissions + await config.api.permission.revoke({ + level: PermissionLevel.READ, + resourceId: viewId, + roleId: "inherited", + }) + return { permissionResource: tableId, triggerResouce: viewId } }, ], From 028d15911ee1d02d17382c7b127d2bbf61935114 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 11:47:08 +0100 Subject: [PATCH 034/195] Revert "Set mssql SHA back to 2019." This reverts commit da805b10ae576f23fd6d8d823d655a3d20382a49. --- packages/server/datasource-sha.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 9b935ed8eb..61249d530c 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d From b0252469ed3a19461452d019093f4fb6df3b0a4d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 12:33:25 +0100 Subject: [PATCH 035/195] Removing wrap for MySQL. --- packages/backend-core/src/sql/sql.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a9400b1839..91b4e124cc 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -991,10 +991,8 @@ class InternalBuilder { this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) }) } - // my-sql needs the where statement to be part of main query, not sub-query - if (sqlClient !== SqlClient.MY_SQL) { - subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) - } + + subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) const standardWrap = (select: string): Knex.QueryBuilder => { subQuery = subQuery.select(`${toAlias}.*`) @@ -1018,11 +1016,7 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = addCorrelatedWhere( - standardWrap(`json_arrayagg(json_object(${fieldList}))`), - isManyToMany ? fromKey! : toKey!, - correlatedFrom - ) + wrapperQuery = knex.raw(`json_arrayagg(json_object(${fieldList}))`) break case SqlClient.ORACLE: wrapperQuery = standardWrap( From c8d1956c73baed7c025fa15b2144cdd7bc309de4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 12:59:20 +0100 Subject: [PATCH 036/195] Fixing AI test cases. --- .../src/automations/tests/openai.spec.ts | 23 ++++++----- yarn.lock | 39 +++++++++++++++---- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 342288a6a1..6b5e1fbb6a 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -23,14 +23,15 @@ jest.mock("openai", () => ({ }, })), })) - jest.mock("@budibase/pro", () => ({ ...jest.requireActual("@budibase/pro"), ai: { - LargeLanguageModel: jest.fn().mockImplementation(() => ({ - init: jest.fn(), - run: jest.fn(), - })), + LargeLanguageModel: { + forCurrentTenant: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, }, features: { isAICustomConfigsEnabled: jest.fn(), @@ -38,6 +39,7 @@ jest.mock("@budibase/pro", () => ({ }, })) +const mockedPro = jest.mocked(pro) const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -121,11 +123,14 @@ describe("test the openai action", () => { prompt, }) - expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( + "gpt-4o-mini" + ) - // @ts-ignore - const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value - expect(llmInstance.init).toHaveBeenCalled() + const llmInstance = + mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) }) diff --git a/yarn.lock b/yarn.lock index 84d64637e4..cd850e833d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12372,10 +12372,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad" - integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w== +"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f" + integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ== dependencies: axios "^1.4.0" lodash "^4.17.21" @@ -20786,7 +20786,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20877,7 +20886,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20891,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22846,7 +22862,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22864,6 +22880,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 8a5a94338ad7b63d83288b73b24358f2a2a420de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 23 Sep 2024 14:18:34 +0100 Subject: [PATCH 037/195] Remove extra logging. --- .github/workflows/budibase_ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 388f2000ed..4b9ebf1e5d 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -186,8 +186,6 @@ jobs: - run: yarn --frozen-lockfile - name: Test server - env: - DEBUG: "testcontainers*" run: | if ${{ env.ONLY_AFFECTED_TASKS }}; then node scripts/run-affected.js --task=test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} From c083fe3bd7fa99d52a26201e65f0b81ae254e498 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 14:43:46 +0100 Subject: [PATCH 038/195] fix issue with js bindings not using quotes --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 927b8588b3..aceb980786 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -643,8 +643,8 @@ runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName - ? `stepsByName[${bindingName}].${name}` - : `steps[${idx - loopBlockCount}].${name}` + ? `stepsByName["${bindingName}"].${name}` + : `steps["${idx - loopBlockCount}"].${name}` } else { runtimeName = hasUserDefinedName ? `stepsByName.${bindingName}.${name}` From 189caa6235b14404f0ccff4a66797ad1bc7497aa Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 14:59:41 +0100 Subject: [PATCH 039/195] fix issue where you could have multiple steps with the same name --- .../FlowChart/FlowItemHeader.svelte | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 5533572511..52f3f49511 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -16,9 +16,11 @@ export let enableNaming = true let validRegex = /^[A-Za-z0-9_\s]+$/ let typing = false + let editing = false const dispatch = createEventDispatcher() $: stepNames = $selectedAutomation?.definition.stepNames + $: allSteps = $selectedAutomation?.definition.steps || [] $: automationName = stepNames?.[block.id] || block?.name || "" $: automationNameError = getAutomationNameError(automationName) $: status = updateStatus(testResult) @@ -56,9 +58,17 @@ } } const getAutomationNameError = name => { - if (stepNames) { + if (stepNames && editing) { + // Check against stepNames for (const [key, value] of Object.entries(stepNames)) { - if (name === value && key !== block.id) { + if (name !== block.name && name === value && key !== block.id) { + return "This name already exists, please enter a unique name" + } + } + + // Check against other block names + for (const step of allSteps) { + if (step.id !== block.id && name === step.name) { return "This name already exists, please enter a unique name" } } @@ -67,11 +77,11 @@ if (name !== block.name && name?.length > 0) { let invalidRoleName = !validRegex.test(name) if (invalidRoleName) { - return "Please enter a role name consisting of only alphanumeric symbols and underscores" + return "Please enter a name consisting of only alphanumeric symbols and underscores" } - - return null } + + return null } const startTyping = async () => { @@ -89,13 +99,28 @@ await automationStore.actions.saveAutomationName(block.id, automationName) } } + + const startEditing = () => { + editing = true + typing = true + } + + const stopEditing = async () => { + editing = false + typing = false + if (automationNameError) { + automationName = stepNames[block.id] || block?.name + } else { + await saveName() + } + }
dispatch("toggle")} > @@ -132,7 +157,7 @@ { e.stopPropagation() - startTyping() + startEditing() }} on:keydown={async e => { if (e.key === "Enter") { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() - } - } - }} - on:blur={async () => { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() + await stopEditing() } }} + on:blur={stopEditing} /> {:else}
@@ -222,7 +235,7 @@ /> {/if}
- {#if automationNameError} + {#if automationNameError && editing}
From 335240718c94ee4e433d4f2c1706f17c86f6524b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:02:37 +0100 Subject: [PATCH 040/195] tidy up --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 52f3f49511..f85496ec81 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -59,14 +59,12 @@ } const getAutomationNameError = name => { if (stepNames && editing) { - // Check against stepNames for (const [key, value] of Object.entries(stepNames)) { if (name !== block.name && name === value && key !== block.id) { return "This name already exists, please enter a unique name" } } - // Check against other block names for (const step of allSteps) { if (step.id !== block.id && name === step.name) { return "This name already exists, please enter a unique name" From 63651b21e6f43f827ee6db38b5b90c47b4fa9f15 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:17:44 +0100 Subject: [PATCH 041/195] lint --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index f85496ec81..361164cfe5 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -82,10 +82,6 @@ return null } - const startTyping = async () => { - typing = true - } - const saveName = async () => { if (automationNameError || block.name === automationName) { return From 89354f640bf30534924ddb1814f2fb4eb681b120 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:29:27 +0100 Subject: [PATCH 042/195] dupe string --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 361164cfe5..a98c597142 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -58,16 +58,18 @@ } } const getAutomationNameError = name => { + const duplicateError = + "This name already exists, please enter a unique name" if (stepNames && editing) { for (const [key, value] of Object.entries(stepNames)) { if (name !== block.name && name === value && key !== block.id) { - return "This name already exists, please enter a unique name" + return duplicateError } } for (const step of allSteps) { if (step.id !== block.id && name === step.name) { - return "This name already exists, please enter a unique name" + return duplicateError } } } From 676058bbbdf47ce79c4b699a3638ea1ec4604bd3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 17:16:34 +0100 Subject: [PATCH 043/195] Updates to limit the response of JSON_ARRAYAGG in mysql/mariaDB - rather than using a limited sub-query which is dis-allowed in MySQL/MariaDB due to the nature of the correlated sub-query. --- packages/backend-core/src/sql/sql.ts | 29 +++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 91b4e124cc..db4ddf180b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -958,22 +958,9 @@ class InternalBuilder { const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) - .limit(getRelationshipLimit()) // add sorting to get consistent order .orderBy(primaryKey) - const addCorrelatedWhere = ( - query: Knex.QueryBuilder, - column1: string, - column2: string - ) => { - return query.where( - column1, - "=", - knex.raw(this.quotedIdentifier(column2)) - ) - } - const isManyToMany = throughTable && toPrimary && fromPrimary let correlatedTo = isManyToMany ? `${throughAlias}.${fromKey}` @@ -992,10 +979,15 @@ class InternalBuilder { }) } - subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) + // add the correlation to the overall query + subQuery = subQuery.where( + correlatedTo, + "=", + knex.raw(this.quotedIdentifier(correlatedFrom)) + ) const standardWrap = (select: string): Knex.QueryBuilder => { - subQuery = subQuery.select(`${toAlias}.*`) + subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, @@ -1016,7 +1008,12 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = knex.raw(`json_arrayagg(json_object(${fieldList}))`) + // can't use the standard wrap due to correlated sub-query limitations in MariaDB + wrapperQuery = subQuery.select( + knex.raw( + `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})` + ) + ) break case SqlClient.ORACLE: wrapperQuery = standardWrap( From 680c68a35b07559b496552c630af41e2c45599aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 18:41:23 +0100 Subject: [PATCH 044/195] Adding test case. --- .../src/api/routes/tests/search.spec.ts | 94 ++++++++++++++----- .../src/sdk/app/tables/external/index.ts | 6 +- .../src/sdk/app/tables/external/utils.ts | 6 +- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 0b0802bab2..4a695edc06 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -42,6 +42,7 @@ import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" import { generateRowIdField } from "../../../integrations/utils" +import { cloneDeep } from "lodash/fp" describe.each([ ["in-memory", undefined], @@ -66,6 +67,35 @@ describe.each([ let table: Table let rows: Row[] + async function basicRelationshipTables(type: RelationshipType) { + const relatedTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: type, + name: "productCat", + fieldName: "product", + tableId: relatedTable._id!, + constraints: { + type: "array", + }, + }, + }, + "product" + ) + return { + relatedTable: await config.api.table.get(relatedTable._id!), + table, + } + } + beforeAll(async () => { await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) if (isLucene) { @@ -201,6 +231,7 @@ describe.each([ // rows returned by the query will also cause the assertion to fail. async toMatchExactly(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -211,7 +242,7 @@ describe.each([ expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) - return response + return cloned } // Asserts that the query returns rows matching exactly the set of rows @@ -219,6 +250,7 @@ describe.each([ // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -231,7 +263,7 @@ describe.each([ ) ) ) - return response + return cloned } // Asserts that the query returns some property values - this cannot be used @@ -239,6 +271,7 @@ describe.each([ // typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...) async toMatch(properties: Record) { const response = await this.performSearch() + const cloned = cloneDeep(response) const keys = Object.keys(properties) as Array> for (let key of keys) { // eslint-disable-next-line jest/no-standalone-expect @@ -248,17 +281,18 @@ describe.each([ expect(response[key]).toEqual(properties[key]) } } - return response + return cloned } // Asserts that the query doesn't return a property, e.g. pagination parameters. async toNotHaveProperty(properties: (keyof SearchResponse)[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) for (let property of properties) { // eslint-disable-next-line jest/no-standalone-expect expect(response[property]).toBeUndefined() } - return response + return cloned } // Asserts that the query returns rows matching the set of rows passed in. @@ -266,6 +300,7 @@ describe.each([ // assertion to fail. async toContain(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -276,7 +311,7 @@ describe.each([ ) ) ) - return response + return cloned } async toFindNothing() { @@ -2196,28 +2231,10 @@ describe.each([ let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - productCategoryTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) - table = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - productCat: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "productCat", - fieldName: "product", - tableId: productCategoryTable._id!, - constraints: { - type: "array", - }, - }, - }, - "product" + const { relatedTable } = await basicRelationshipTables( + RelationshipType.ONE_TO_MANY ) + productCategoryTable = relatedTable productCatRows = await Promise.all([ config.api.row.save(productCategoryTable._id!, { name: "foo" }), @@ -2262,6 +2279,31 @@ describe.each([ }).toContainExactly([{ name: "baz", productCat: undefined }]) }) }) + + isSql && + describe("big relations", () => { + beforeAll(async () => { + const { relatedTable } = await basicRelationshipTables( + RelationshipType.MANY_TO_ONE + ) + const mainRow = await config.api.row.save(table._id!, { + name: "foo", + }) + for (let i = 0; i < 11; i++) { + await config.api.row.save(relatedTable._id!, { + name: i, + product: [mainRow._id!], + }) + } + }) + + it("can only pull 500 related rows", async () => { + await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(10) + }) + }) + }) ;(isSqs || isLucene) && describe("relations to same table", () => { let relatedTable: Table, relatedRows: Row[] diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 842b6b5648..913eae6d1f 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -198,6 +198,7 @@ export async function save( } } generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) + tables[relatedTable.name] = relatedTable schema.main = true } @@ -231,7 +232,10 @@ export async function save( // remove the rename prop delete tableToSave._rename - datasource.entities[tableToSave.name] = tableToSave + datasource.entities = { + ...datasource.entities, + ...tables, + } // store it into couch now for budibase reference await db.put(populateExternalTableSchemas(datasource)) diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 321bd990f5..f27c59dc5a 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -22,12 +22,16 @@ export function cleanupRelationships( tables: Record, oldTable?: Table ) { + if (!oldTable) { + return + } const tableToIterate = oldTable ? oldTable : table // clean up relationships in couch table schemas for (let [key, schema] of Object.entries(tableToIterate.schema)) { if ( schema.type === FieldType.LINK && - (!oldTable || table.schema[key] == null) + oldTable.schema[key] != null && + table.schema[key] == null ) { const schemaTableId = schema.tableId const relatedTable = Object.values(tables).find( From 980615b37b5b83b109278a012093ee06e9bc0e19 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 11:08:44 +0100 Subject: [PATCH 045/195] looping query rows --- .../tests/scenarios/looping.spec.ts | 146 ++++++++++++++++++ .../tests/scenarios/scenarios.spec.ts | 2 +- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9bc382a187..9f7ff04156 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -5,6 +5,7 @@ import { LoopStepType, CreateRowStepOutputs, ServerLogStepOutputs, + FieldType, } from "@budibase/types" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" @@ -242,4 +243,149 @@ describe("Loop automations", () => { expect(results.steps[1].outputs.message).toContain("- 3") expect(results.steps[3].outputs.message).toContain("- 3") }) + + it("should run an automation with a loop and update row step", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const rows = [ + { name: "Row 1", value: 1, tableId: table._id }, + { name: "Row 2", value: 2, tableId: table._id }, + { name: "Row 3", value: 3, tableId: table._id }, + ] + + for (const row of rows) { + await config.createRow(row) + } + + const builder = createAutomationBuilder({ + name: "Test Loop and Update Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.rows }}", + }) + .updateRow({ + rowId: "{{ loop.currentItem._id }}", + row: { + name: "Updated {{ loop.currentItem.name }}", + value: "{{ loop.currentItem.value }}", + tableId: table._id, + }, + meta: {}, + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + const expectedRows = [ + { name: "Updated Row 1", value: 1 }, + { name: "Updated Row 2", value: 2 }, + { name: "Updated Row 3", value: 3 }, + ] + + expect(results.steps[1].outputs.items).toEqual( + expect.arrayContaining( + expectedRows.map(row => + expect.objectContaining({ + success: true, + row: expect.objectContaining(row), + }) + ) + ) + ) + + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining( + expectedRows.map(row => expect.objectContaining(row)) + ) + ) + + expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length) + expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length) + }) + + it("should run an automation with a loop and delete row step", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const rows = [ + { name: "Row 1", value: 1, tableId: table._id }, + { name: "Row 2", value: 2, tableId: table._id }, + { name: "Row 3", value: 3, tableId: table._id }, + ] + + for (const row of rows) { + await config.createRow(row) + } + + const builder = createAutomationBuilder({ + name: "Test Loop and Delete Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.rows }}", + }) + .deleteRow({ + tableId: table._id!, + id: "{{ loop.currentItem._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[2].outputs.rows).toHaveLength(0) + }) }) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 62a1b9db8f..12b8392ffc 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -197,7 +197,7 @@ describe("Automation Scenarios", () => { ) }) }) - describe.only("Automations with filter", () => { + describe("Automations with filter", () => { let table: Table beforeEach(async () => { From 956df101e84891aeef9e7c49a22772e03ae33ef9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 11:16:42 +0100 Subject: [PATCH 046/195] PR comments and type improvements. --- .../src/api/routes/tests/search.spec.ts | 8 +++++++- .../src/sdk/app/tables/external/index.ts | 6 ++++-- .../src/sdk/app/tables/external/utils.ts | 20 +++++++++++++------ .../server/src/tests/utilities/structures.ts | 10 ++++------ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 4a695edc06..4125d7bf5b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -77,6 +77,7 @@ describe.each([ table = await createTable( { name: { name: "name", type: FieldType.STRING }, + //@ts-ignore - API accepts this structure, will build out rest of definition productCat: { type: FieldType.LINK, relationshipType: type, @@ -2297,12 +2298,17 @@ describe.each([ } }) - it("can only pull 500 related rows", async () => { + it("can only pull 10 related rows", async () => { await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { const response = await expectQuery({}).toContain([{ name: "foo" }]) expect(response.rows[0].productCat).toBeArrayOfSize(10) }) }) + + it("can pull max rows when env not set (defaults to 500)", async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(11) + }) }) ;(isSqs || isLucene) && describe("relations to same table", () => { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 913eae6d1f..e374e70c87 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -204,7 +204,9 @@ export async function save( // add in the new table for relationship purposes tables[tableToSave.name] = tableToSave - cleanupRelationships(tableToSave, tables, oldTable) + if (oldTable) { + cleanupRelationships(tableToSave, tables, { oldTable }) + } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE await makeTableRequest( @@ -259,7 +261,7 @@ export async function destroy(datasourceId: string, table: Table) { const operation = Operation.DELETE_TABLE if (tables) { await makeTableRequest(datasource, operation, table, tables) - cleanupRelationships(table, tables) + cleanupRelationships(table, tables, { deleting: true }) delete tables[table.name] datasource.entities = tables } diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index f27c59dc5a..21ffa21053 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -20,17 +20,25 @@ import { cloneDeep } from "lodash/fp" export function cleanupRelationships( table: Table, tables: Record, - oldTable?: Table -) { - if (!oldTable) { - return - } + opts: { oldTable: Table } +): void +export function cleanupRelationships( + table: Table, + tables: Record, + opts: { deleting: boolean } +): void +export function cleanupRelationships( + table: Table, + tables: Record, + opts?: { oldTable?: Table; deleting?: boolean } +): void { + const oldTable = opts?.oldTable const tableToIterate = oldTable ? oldTable : table // clean up relationships in couch table schemas for (let [key, schema] of Object.entries(tableToIterate.schema)) { if ( schema.type === FieldType.LINK && - oldTable.schema[key] != null && + (opts?.deleting || oldTable?.schema[key] != null) && table.schema[key] == null ) { const schemaTableId = schema.tableId diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 2e501932b8..72cd31e383 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -600,10 +600,10 @@ export function fullSchemaWithoutLinks({ allRequired, }: { allRequired?: boolean -}) { - const schema: { - [type in Exclude]: FieldSchema & { type: type } - } = { +}): { + [type in Exclude]: FieldSchema & { type: type } +} { + return { [FieldType.STRING]: { name: "string", type: FieldType.STRING, @@ -741,8 +741,6 @@ export function fullSchemaWithoutLinks({ }, }, } - - return schema } export function basicAttachment() { return { From 6a7959e93c4c6ece9e62ce0402e80ab522a923ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 11:36:14 +0100 Subject: [PATCH 047/195] Fixing test case. --- .../src/api/routes/tests/search.spec.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 4125d7bf5b..706151ba2c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -39,20 +39,20 @@ import tk from "timekeeper" import { encodeJSBinding } from "@budibase/string-templates" import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" -import { structures } from "@budibase/backend-core/tests" +import { generator, structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], - ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // ["in-memory", undefined], + // ["lucene", undefined], + // ["sqs", undefined], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -72,7 +72,7 @@ describe.each([ { name: { name: "name", type: FieldType.STRING }, }, - "productCategory" + generator.guid().substring(0, 10) ) table = await createTable( { @@ -89,7 +89,7 @@ describe.each([ }, }, }, - "product" + generator.guid().substring(0, 10) ) return { relatedTable: await config.api.table.get(relatedTable._id!), @@ -2268,7 +2268,7 @@ describe.each([ it("should be able to filter by relationship using table name", async () => { await expectQuery({ - equal: { ["productCategory.name"]: "foo" }, + equal: { [`${productCategoryTable.name}.name`]: "foo" }, }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, ]) From 464f973f122154c2c3169abbd4484417b53c355a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 12:01:47 +0100 Subject: [PATCH 048/195] Adding a separation for MariaDB and MySQL, mariaDB is the core of the problem, this solves for it by separating them and allowing us to use the special json_arrayagg for mariaDB, but use a correlated sub-query for MySQL. --- packages/backend-core/src/sql/sql.ts | 20 +++++++++++++------ packages/backend-core/src/sql/sqlTable.ts | 13 +++++++++++- .../src/api/routes/tests/search.spec.ts | 14 ++++++------- packages/server/src/integrations/mysql.ts | 10 ++++++++++ packages/types/src/sdk/search.ts | 1 + 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index db4ddf180b..4bdec363a4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -150,6 +150,7 @@ class InternalBuilder { return `"${str}"` case SqlClient.MS_SQL: return `[${str}]` + case SqlClient.MARIADB: case SqlClient.MY_SQL: return `\`${str}\`` } @@ -559,7 +560,10 @@ class InternalBuilder { )}${wrap}, FALSE)` ) }) - } else if (this.client === SqlClient.MY_SQL) { + } else if ( + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB + ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (q, key, value) => { return q[rawFnc]( @@ -1007,7 +1011,7 @@ class InternalBuilder { `json_agg(json_build_object(${fieldList}))` ) break - case SqlClient.MY_SQL: + case SqlClient.MARIADB: // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( knex.raw( @@ -1015,6 +1019,7 @@ class InternalBuilder { ) ) break + case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -1181,7 +1186,8 @@ class InternalBuilder { if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || - this.client === SqlClient.MY_SQL + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB ) { const primary = this.table.primary if (!primary) { @@ -1328,12 +1334,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { - client: sqlClient, + client: this.getBaseSqlClient(), } if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } - const client = knex(config) let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient, client, json) @@ -1442,7 +1447,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { let id if (sqlClient === SqlClient.MS_SQL) { id = results?.[0].id - } else if (sqlClient === SqlClient.MY_SQL) { + } else if ( + sqlClient === SqlClient.MY_SQL || + sqlClient === SqlClient.MARIADB + ) { id = results?.insertId } row = processFn( diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 35d7978449..f5b02cc4e4 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -210,16 +210,27 @@ function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder { class SqlTableQueryBuilder { private readonly sqlClient: SqlClient + private extendedSqlClient: SqlClient | undefined // pass through client to get flavour of SQL constructor(client: SqlClient) { this.sqlClient = client } - getSqlClient(): SqlClient { + getBaseSqlClient(): SqlClient { return this.sqlClient } + getSqlClient(): SqlClient { + return this.extendedSqlClient || this.sqlClient + } + + // if working in a database like MySQL with many variants (MariaDB) + // we can set another client which overrides the base one + setExtendedSqlClient(client: SqlClient) { + this.extendedSqlClient = client + } + /** * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 706151ba2c..c770c4e460 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + ["in-memory", undefined], + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f5b575adb8..8b1ada4184 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -241,6 +241,16 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async connect() { this.client = await mysql.createConnection(this.config) + const res = await this.internalQuery( + { + sql: "SELECT VERSION();", + }, + { connect: false } + ) + const version = res?.[0]?.["VERSION()"] + if (version?.toLowerCase().includes("mariadb")) { + this.setExtendedSqlClient(SqlClient.MARIADB) + } } async disconnect() { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 7d61aebdfb..7c691beb71 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -195,6 +195,7 @@ export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", MY_SQL = "mysql2", + MARIADB = "mariadb", ORACLE = "oracledb", SQL_LITE = "sqlite3", } From 3e9ca562c598a7099ba9c18669b8d6cda8f37013 Mon Sep 17 00:00:00 2001 From: mikesealey Date: Tue, 24 Sep 2024 12:12:46 +0100 Subject: [PATCH 049/195] reworks changes based on feedback --- packages/bbui/src/Form/Core/Dropzone.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index e223d789a1..2922d88e7a 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -397,9 +397,10 @@ margin-bottom: 8px; } - .compact .gallery > * { - max-height: 25px; + .compact .placeholder { + height: fit-content; } + .title { display: flex; flex-direction: row; From 51774b3434a344e0ab8e766b96ad2e840ce44033 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 12:30:45 +0100 Subject: [PATCH 050/195] Working on plumbing 'source' all the way through our code. --- .../backend-core/src/context/mainContext.ts | 19 +++- packages/backend-core/src/context/types.ts | 3 +- packages/backend-core/src/docIds/params.ts | 14 +-- packages/backend-core/src/sql/sql.ts | 3 - .../api/controllers/row/ExternalRequest.ts | 89 ++++++++++--------- .../src/api/controllers/row/external.ts | 19 ++-- .../src/api/controllers/row/utils/utils.ts | 13 ++- .../server/src/api/controllers/row/views.ts | 7 +- packages/server/src/db/utils.ts | 8 +- .../middleware/triggerRowActionAuthorised.ts | 2 +- .../server/src/sdk/app/permissions/index.ts | 6 +- packages/server/src/sdk/app/rowActions.ts | 6 +- .../server/src/sdk/app/rows/queryUtils.ts | 7 +- packages/server/src/sdk/app/rows/rows.ts | 5 +- packages/server/src/sdk/app/rows/search.ts | 58 +++++++----- .../src/sdk/app/rows/search/external.ts | 7 +- .../server/src/sdk/app/rows/search/utils.ts | 6 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 20 ++--- packages/server/src/sdk/app/rows/utils.ts | 35 +++----- packages/server/src/sdk/app/tables/utils.ts | 4 + packages/server/src/sdk/app/views/index.ts | 19 +++- packages/types/src/sdk/row.ts | 5 +- yarn.lock | 64 +------------ 23 files changed, 204 insertions(+), 215 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index a52a17dd53..25b273e51c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext, Snippet, App } from "@budibase/types" +import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record) { context.featureFlagCache ??= {} context.featureFlagCache[key] = value } + +export function getTableForView(viewId: string): Table | undefined { + const context = getCurrentContext() + if (!context) { + return + } + return context.viewToTableCache?.[viewId] +} + +export function setTableForView(viewId: string, table: Table) { + const context = getCurrentContext() + if (!context) { + return + } + context.viewToTableCache ??= {} + context.viewToTableCache[viewId] = table +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index fe6072e85c..ee84b49459 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, Snippet, VM } from "@budibase/types" +import { IdentityContext, Snippet, Table, VM } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { GoogleSpreadsheet } from "google-spreadsheet" @@ -21,4 +21,5 @@ export type ContextMap = { featureFlagCache?: { [key: string]: Record } + viewToTableCache?: Record } diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 093724b55e..016604b69b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,7 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" -import { DatabaseQueryOpts } from "@budibase/types" +import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) { /** * Check if a given ID is that of a table. - * @returns {boolean} */ -export const isTableId = (id: string) => { +export const isTableId = (id: string): boolean => { // this includes datasource plus tables return ( !!id && @@ -77,13 +76,16 @@ export const isTableId = (id: string) => { ) } +export function isViewId(id: string): boolean { + return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`) +} + /** * Check if a given ID is that of a datasource or datasource plus. - * @returns {boolean} */ -export const isDatasourceId = (id: string) => { +export const isDatasourceId = (id: string): boolean => { // this covers both datasources and datasource plus - return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) + return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) } /** diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 023bfd5d8a..97fc4124fb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1274,9 +1274,6 @@ class InternalBuilder { if (counting) { query = this.addDistinctCount(query) } else if (aggregations.length > 0) { - query = query.select( - this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") - ) query = this.addAggregations(query, aggregations) } else { query = query.select(this.generateSelectStatement()) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 9c9bd0b284..a52d7abcd1 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -19,6 +19,7 @@ import { SortJson, SortType, Table, + ViewV2, } from "@budibase/types" import { breakExternalTableId, @@ -159,17 +160,41 @@ function isEditableColumn(column: FieldSchema) { export class ExternalRequest { private readonly operation: T - private readonly tableId: string - private datasource?: Datasource - private tables: { [key: string]: Table } = {} + private readonly source: Table | ViewV2 + private datasource: Datasource - constructor(operation: T, tableId: string, datasource?: Datasource) { - this.operation = operation - this.tableId = tableId - this.datasource = datasource - if (datasource && datasource.entities) { - this.tables = datasource.entities + public static async for( + operation: T, + source: Table | ViewV2, + opts: { datasource?: Datasource } = {} + ) { + if (!opts.datasource) { + if (sdk.views.isView(source)) { + const table = await sdk.views.getTable(source.id) + opts.datasource = await sdk.datasources.get(table.sourceId!) + } else { + opts.datasource = await sdk.datasources.get(source.sourceId!) + } } + + return new ExternalRequest(operation, source, opts.datasource) + } + + private get tables(): { [key: string]: Table } { + if (!this.datasource.entities) { + throw new Error("Datasource does not have entities") + } + return this.datasource.entities + } + + private constructor( + operation: T, + source: Table | ViewV2, + datasource: Datasource + ) { + this.operation = operation + this.source = source + this.datasource = datasource } private prepareFilters( @@ -290,20 +315,6 @@ export class ExternalRequest { return this.tables[tableName] } - // seeds the object with table and datasource information - async retrieveMetadata( - datasourceId: string - ): Promise<{ tables: Record; datasource: Datasource }> { - if (!this.datasource) { - this.datasource = await sdk.datasources.get(datasourceId) - if (!this.datasource || !this.datasource.entities) { - throw "No tables found, fetch tables before query." - } - this.tables = this.datasource.entities - } - return { tables: this.tables, datasource: this.datasource } - } - async getRow(table: Table, rowId: string): Promise { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), @@ -619,24 +630,16 @@ export class ExternalRequest { } async run(config: RunConfig): Promise> { - const { operation, tableId } = this - if (!tableId) { - throw new Error("Unable to run without a table ID") - } - let { datasourceId, tableName } = breakExternalTableId(tableId) - let datasource = this.datasource - if (!datasource) { - const { datasource: ds } = await this.retrieveMetadata(datasourceId) - datasource = ds - } - const tables = this.tables - const table = tables[tableName] - let isSql = isSQL(datasource) - if (!table) { - throw new Error( - `Unable to process query, table "${tableName}" not defined.` - ) + const { operation } = this + let table: Table + if (sdk.views.isView(this.source)) { + table = await sdk.views.getTable(this.source.id) + } else { + table = this.source } + + let isSql = isSQL(this.datasource) + // look for specific components of config which may not be considered acceptable let { id, row, filters, sort, paginate, rows } = cleanupConfig( config, @@ -687,8 +690,8 @@ export class ExternalRequest { } let json: QueryJson = { endpoint: { - datasourceId: datasourceId!, - entityId: tableName, + datasourceId: this.datasource._id!, + entityId: table.name, operation, }, resource: { @@ -714,7 +717,7 @@ export class ExternalRequest { }, meta: { table, - tables: tables, + tables: this.tables, }, } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index bd5201c05c..fb992059f4 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -17,6 +17,7 @@ import { Row, Table, UserCtx, + ViewV2, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" @@ -29,29 +30,29 @@ import { generateIdForRow } from "./utils" export async function handleRequest( operation: T, - tableId: string, + source: Table | ViewV2, opts?: RunConfig ): Promise> { - return new ExternalRequest(operation, tableId, opts?.datasource).run( - opts || {} - ) + return ( + await ExternalRequest.for(operation, source, { + datasource: opts?.datasource, + }) + ).run(opts || {}) } export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) - + const source = await utils.getSource(ctx) const { _id, ...rowData } = ctx.request.body - const table = await sdk.tables.getTable(tableId) const { row: dataToUpdate } = await inputProcessing( ctx.user?._id, - cloneDeep(table), + cloneDeep(source), rowData ) const validateResult = await sdk.rows.utils.validate({ row: dataToUpdate, - tableId, + source, }) if (!validateResult.valid) { throw { validation: validateResult.errors } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 1fc7221294..91c0fc966f 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context } from "@budibase/backend-core" +import { context, docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -9,6 +9,7 @@ import { RelationshipsJson, Row, Table, + ViewV2, } from "@budibase/types" import { processDates, @@ -78,7 +79,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { const { sourceId } = ctx.params - if (utils.isViewID(sourceId)) { + if (docIds.isViewId(sourceId)) { return { tableId: utils.extractViewInfoFromID(sourceId).tableId, viewId: sourceId, @@ -97,6 +98,14 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { throw new Error("Unable to find table ID in request") } +export async function getSource(ctx: Ctx): Promise { + const { tableId, viewId } = getSourceId(ctx) + if (viewId) { + return sdk.views.get(viewId) + } + return sdk.tables.getTable(tableId) +} + export async function validate( opts: { row: Row } & ({ tableId: string } | { table: Table }) ) { diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f9565f0e82..7008c5e0be 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -84,11 +84,8 @@ export async function searchView( })) const searchOptions: RequiredKeys & - RequiredKeys< - Pick - > = { - tableId: view.tableId, - viewId: view.id, + RequiredKeys> = { + sourceId: view.id, query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 043394e7a6..6c1065e847 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,4 +1,4 @@ -import { context, db as dbCore, utils } from "@budibase/backend-core" +import { context, db as dbCore, docIds, utils } from "@budibase/backend-core" import { DatabaseQueryOpts, Datasource, @@ -318,12 +318,8 @@ export function generateViewID(tableId: string) { }${SEPARATOR}${tableId}${SEPARATOR}${newid()}` } -export function isViewID(viewId: string) { - return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW -} - export function extractViewInfoFromID(viewId: string) { - if (!isViewID(viewId)) { + if (!docIds.isViewId(viewId)) { throw new Error("Unable to extract table ID, is not a view ID") } const split = viewId.split(SEPARATOR) diff --git a/packages/server/src/middleware/triggerRowActionAuthorised.ts b/packages/server/src/middleware/triggerRowActionAuthorised.ts index 17f22b7000..3e903ba907 100644 --- a/packages/server/src/middleware/triggerRowActionAuthorised.ts +++ b/packages/server/src/middleware/triggerRowActionAuthorised.ts @@ -15,7 +15,7 @@ export function triggerRowActionAuthorised( const rowActionId: string = ctx.params[actionPath] const isTableId = docIds.isTableId(sourceId) - const isViewId = utils.isViewID(sourceId) + const isViewId = docIds.isViewId(sourceId) if (!isTableId && !isViewId) { ctx.throw(400, `'${sourceId}' is not a valid source id`) } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index a6e81652ee..d5e4aefe3a 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,10 +1,10 @@ -import { db, roles } from "@budibase/backend-core" +import { db, docIds, roles } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, } from "@budibase/types" -import { extractViewInfoFromID, isViewID } from "../../../db/utils" +import { extractViewInfoFromID } from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, @@ -20,7 +20,7 @@ type ResourcePermissions = Record< export async function getInheritablePermissions( resourceId: string ): Promise { - if (isViewID(resourceId)) { + if (docIds.isViewId(resourceId)) { return await getResourcePerms(extractViewInfoFromID(resourceId).tableId) } } diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts index 9a7d402df0..21c256eacb 100644 --- a/packages/server/src/sdk/app/rowActions.ts +++ b/packages/server/src/sdk/app/rowActions.ts @@ -1,11 +1,11 @@ -import { context, HTTPError, utils } from "@budibase/backend-core" +import { context, docIds, HTTPError, utils } from "@budibase/backend-core" import { AutomationTriggerStepId, SEPARATOR, TableRowActions, VirtualDocumentType, } from "@budibase/types" -import { generateRowActionsID, isViewID } from "../../db/utils" +import { generateRowActionsID } from "../../db/utils" import automations from "./automations" import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo" import * as triggers from "../../automations/triggers" @@ -155,7 +155,7 @@ export async function update( async function guardView(tableId: string, viewId: string) { let view - if (isViewID(viewId)) { + if (docIds.isViewId(viewId)) { view = await sdk.views.get(viewId) } if (!view || view.tableId !== tableId) { diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 65f400a1d9..a88763215e 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -53,8 +53,8 @@ export const removeInvalidFilters = ( } export const getQueryableFields = async ( - fields: string[], - table: Table + table: Table, + fields?: string[] ): Promise => { const extractTableFields = async ( table: Table, @@ -110,6 +110,9 @@ export const getQueryableFields = async ( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] + if (fields === undefined) { + fields = Object.keys(table.schema) + } result.push(...(await extractTableFields(table, fields, [table._id!]))) return result diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index c61b8692ed..ef25d06baf 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,9 +1,8 @@ -import { db as dbCore, context } from "@budibase/backend-core" +import { db as dbCore, context, docIds } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" import { extractViewInfoFromID, getRowParams, - isViewID, } from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" @@ -26,7 +25,7 @@ export async function getAllInternalRows(appId?: string) { function pickApi(tableOrViewId: string) { let tableId = tableOrViewId - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { tableId = extractViewInfoFromID(tableOrViewId).tableId } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index dbf0cefd51..fa01a6cf13 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -4,6 +4,8 @@ import { RowSearchParams, SearchResponse, SortOrder, + Table, + ViewV2, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -12,7 +14,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features } from "@budibase/backend-core" +import { features, docIds } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" @@ -36,8 +38,7 @@ export async function search( ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ - tableId: options.tableId, - viewId: options.viewId, + sourceId: options.sourceId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, @@ -52,20 +53,18 @@ export async function search( .join(", "), }) - const isExternalTable = isExternalTableID(options.tableId) options.query = dataFilters.cleanupQuery(options.query || {}) options.query = dataFilters.fixupFilterArrays(options.query) - span?.addTags({ + span.addTags({ cleanedQuery: options.query, - isExternalTable, }) if ( !dataFilters.hasFilters(options.query) && options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE ) { - span?.addTags({ emptyQuery: true }) + span.addTags({ emptyQuery: true }) return { rows: [], } @@ -75,34 +74,47 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - const table = await sdk.tables.getTable(options.tableId) - options = searchInputMapping(table, options) + let source: Table | ViewV2 + let table: Table + if (docIds.isTableId(options.sourceId)) { + source = await sdk.tables.getTable(options.sourceId) + table = source + options = searchInputMapping(source, options) + } else if (docIds.isViewId(options.sourceId)) { + source = await sdk.views.get(options.sourceId) + table = await sdk.tables.getTable(source.tableId) + options = searchInputMapping(table, options) - if (options.query) { - const tableFields = Object.keys(table.schema).filter( - f => table.schema[f].visible !== false - ) - - const queriableFields = await getQueryableFields( - options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, - table - ) - options.query = removeInvalidFilters(options.query, queriableFields) + span.addTags({ + tableId: table._id, + }) + } else { + throw new Error(`Invalid source ID: ${options.sourceId}`) } + if (options.query) { + const visibleFields = ( + options.fields || Object.keys(table.schema) + ).filter(field => table.schema[field].visible) + + const queryableFields = await getQueryableFields(table, visibleFields) + options.query = removeInvalidFilters(options.query, queryableFields) + } + + const isExternalTable = isExternalTableID(table._id!) let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) - result = await external.search(options, table) + result = await external.search(options, source) } else if (await features.flags.isEnabled("SQS")) { span?.addTags({ searchType: "sqs" }) - result = await internal.sqs.search(options, table) + result = await internal.sqs.search(options, source) } else { span?.addTags({ searchType: "lucene" }) - result = await internal.lucene.search(options, table) + result = await internal.lucene.search(options, source) } - span?.addTags({ + span.addTags({ foundRows: result.rows.length, totalRows: result.totalRows, }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 0ff25a00e4..5584b3f110 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -9,6 +9,7 @@ import { SortJson, SortOrder, Table, + ViewV2, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -60,9 +61,8 @@ function getPaginationAndLimitParameters( export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = @@ -112,10 +112,9 @@ export async function search( : Promise.resolve(undefined), ]) - let processed = await outputProcessing(table, rows, { + let processed = await outputProcessing(source, rows, { preserveLinks: true, squash: true, - fromViewId: options.viewId, }) let hasNextPage = false diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 5ffc065353..d727e58887 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -9,6 +9,7 @@ import { SearchResponse, Row, RowSearchParams, + ViewV2, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" @@ -83,10 +84,7 @@ function userColumnMapping(column: string, options: RowSearchParams) { // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: RowSearchParams) { - if (!table?.schema) { - return options - } - for (let [key, column] of Object.entries(table.schema)) { + for (let [key, column] of Object.entries(table.schema || {})) { switch (column.type) { case FieldType.BB_REFERENCE_SINGLE: { const subtype = column.subtype diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index aabc359484..f399801f1e 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -203,7 +203,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name", "age"]) }) @@ -216,7 +216,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name"]) }) @@ -245,7 +245,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -282,7 +282,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) }) @@ -313,7 +313,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name"]) }) @@ -381,7 +381,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -398,7 +398,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 1 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux1.schema), aux1) + return getQueryableFields(aux1) }) expect(result).toEqual([ "_id", @@ -420,7 +420,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 2 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux2.schema), aux2) + return getQueryableFields(aux2) }) expect(result).toEqual([ "_id", @@ -474,7 +474,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -488,7 +488,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux.schema), aux) + return getQueryableFields(aux) }) expect(result).toEqual([ "_id", diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bc09116b3b..3899009f13 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -13,16 +13,14 @@ import { TableSchema, SqlClient, ArrayOperator, + ViewV2, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." -import { - extractViewInfoFromID, - isRelationshipColumn, - isViewID, -} from "../../../db/utils" +import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" +import { docIds } from "@budibase/backend-core" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -142,37 +140,32 @@ function isForeignKey(key: string, table: Table) { } export async function validate({ - tableId, + source, row, - table, }: { - tableId?: string + source: Table | ViewV2 row: Row - table?: Table }): Promise<{ valid: boolean errors: Record }> { - let fetchedTable: Table | undefined - if (!table && tableId) { - fetchedTable = await sdk.tables.getTable(tableId) - } else if (table) { - fetchedTable = table - } - if (fetchedTable === undefined) { - throw new Error("Unable to fetch table for validation") + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source } const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, FieldType.BB_REFERENCE_SINGLE, ] - for (let fieldName of Object.keys(fetchedTable.schema)) { - const column = fetchedTable.schema[fieldName] + for (let fieldName of Object.keys(table.schema)) { + const column = table.schema[fieldName] const constraints = cloneDeep(column.constraints) const type = column.type // foreign keys are likely to be enriched - if (isForeignKey(fieldName, fetchedTable)) { + if (isForeignKey(fieldName, table)) { continue } // formulas shouldn't validated, data will be deleted anyway @@ -323,7 +316,7 @@ export function isArrayFilter(operator: any): operator is ArrayOperator { } export function tryExtractingTableAndViewId(tableOrViewId: string) { - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { return { tableId: extractViewInfoFromID(tableOrViewId).tableId, viewId: tableOrViewId, diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index b8e3d888af..7a8096fb0a 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -9,3 +9,7 @@ export function isExternal(opts: { table?: Table; tableId?: string }): boolean { } return false } + +export function isTable(table: any): table is Table { + return table.type === "table" +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index de3579f7fd..f911385dc1 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -9,7 +9,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -40,6 +40,23 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +export async function getTable(viewId: string): Promise
{ + const cached = context.getTableForView(viewId) + if (cached) { + return cached + } + const { tableId } = utils.extractViewInfoFromID(viewId) + const table = await sdk.tables.getTable(tableId) + context.setTableForView(viewId, table) + return table +} + +export function isView(view: any): view is ViewV2 { + return ( + view.version === 2 && "id" in view && "tableId" in view && "name" in view + ) +} + async function guardCalculationViewSchema( table: Table, view: Omit diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index f81d56c082..2b6ff3a6c6 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -10,8 +10,7 @@ export interface Aggregation { } export interface SearchParams { - tableId?: string - viewId?: string + sourceId?: string query?: SearchFilters paginate?: boolean bookmark?: string | number @@ -30,7 +29,7 @@ export interface SearchParams { // when searching for rows we want a more extensive search type that requires certain properties export interface RowSearchParams - extends WithRequired {} + extends WithRequired {} export interface SearchResponse { rows: T[] diff --git a/yarn.lock b/yarn.lock index cd850e833d..01466f66b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17796,21 +17796,11 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" -pg-cloudflare@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" - integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== - pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== -pg-connection-string@^2.6.4: - version "2.6.4" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d" - integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA== - pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17821,21 +17811,11 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== -pg-pool@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2" - integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg== - pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== -pg-protocol@^1.6.1: - version "1.6.1" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" - integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== - pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17860,19 +17840,6 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" -pg@^8.12.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79" - integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ== - dependencies: - pg-connection-string "^2.6.4" - pg-pool "^3.6.2" - pg-protocol "^1.6.1" - pg-types "^2.1.0" - pgpass "1.x" - optionalDependencies: - pg-cloudflare "^1.1.1" - pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20786,16 +20753,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20886,7 +20844,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20900,13 +20858,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22862,7 +22813,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22880,15 +22831,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 5f5b38d9a440b9e593e61638de43e48feb7ff27d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 12:31:56 +0100 Subject: [PATCH 051/195] Update the version of PostHog used in the cli package. --- packages/cli/package.json | 2 +- packages/cli/src/analytics/Client.ts | 2 +- yarn.lock | 101 ++------------------------- 3 files changed, 7 insertions(+), 98 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index c9ff373142..8efdcc7816 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,7 +28,7 @@ "inquirer": "8.0.0", "lookpath": "1.1.0", "node-fetch": "2.6.7", - "posthog-node": "1.3.0", + "posthog-node": "4.0.1", "pouchdb": "7.3.0", "@budibase/pouchdb-replication-stream": "1.2.11", "randomstring": "1.1.5", diff --git a/packages/cli/src/analytics/Client.ts b/packages/cli/src/analytics/Client.ts index 19b171026d..0d7f7fea8f 100644 --- a/packages/cli/src/analytics/Client.ts +++ b/packages/cli/src/analytics/Client.ts @@ -1,4 +1,4 @@ -import PostHog from "posthog-node" +import { PostHog } from "posthog-node" import { POSTHOG_TOKEN, AnalyticsEvent } from "../constants" import { ConfigManager } from "../structures/ConfigManager" diff --git a/yarn.lock b/yarn.lock index cd850e833d..0fe11b1f76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7535,15 +7535,7 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -axios-retry@^3.1.9: - version "3.4.0" - resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.4.0.tgz#f464dbe9408e5aa78fa319afd38bb69b533d8854" - integrity sha512-VdgaP+gHH4iQYCCNUWF2pcqeciVOdGrBBAYUfTY+wPcO5Ltvp/37MLFNCmJKo7Gj3SHvCSdL8ouI1qLYJN3liA== - dependencies: - "@babel/runtime" "^7.15.4" - is-retry-allowed "^2.2.0" - -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -8396,11 +8388,6 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== -charenc@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" - integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== - cheap-watch@^1.0.2, cheap-watch@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/cheap-watch/-/cheap-watch-1.0.4.tgz#0bcb4a3a8fbd9d5327936493f6b56baa668d8fef" @@ -8787,11 +8774,6 @@ component-emitter@^1.3.0: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== -component-type@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9" - integrity sha512-Kgy+2+Uwr75vAi6ChWXgHuLvd+QLD7ssgpaRq2zCvt80ptvAfMc/hijcJxXkBa2wMlEZcJvC2H8Ubo+A9ATHIg== - compress-commons@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.2.tgz#6542e59cb63e1f46a8b21b0e06f9a32e4c8b06df" @@ -9210,11 +9192,6 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" -crypt@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" - integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== - crypto-browserify@^3.11.0: version "3.12.0" resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" @@ -13257,11 +13234,6 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@~1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== - is-builtin-module@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" @@ -13546,11 +13518,6 @@ is-retry-allowed@^1.1.0: resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== -is-retry-allowed@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d" - integrity sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg== - is-self-closing@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-self-closing/-/is-self-closing-1.0.1.tgz#5f406b527c7b12610176320338af0fa3896416e4" @@ -14307,11 +14274,6 @@ joi@^17.13.1: "@sideway/formula" "^3.0.1" "@sideway/pinpoint" "^2.0.0" -join-component@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" - integrity sha512-bF7vcQxbODoGK1imE2P9GS9aw4zD0Sd+Hni68IMZLj7zRnquH7dXUmMw9hDI5S/Jzt7q+IyTXN0rSg2GI0IKhQ== - joycon@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" @@ -15801,15 +15763,6 @@ md5.js@^1.3.4: inherits "^2.0.1" safe-buffer "^5.1.2" -md5@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" - integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== - dependencies: - charenc "0.0.2" - crypt "0.0.2" - is-buffer "~1.1.6" - mdn-data@2.0.14: version "2.0.14" resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" @@ -16271,7 +16224,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -18440,20 +18393,6 @@ posthog-js@^1.13.4: preact "^10.19.3" web-vitals "^4.0.1" -posthog-node@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.3.0.tgz#804ed2f213a2f05253f798bf9569d55a9cad94f7" - integrity sha512-2+VhqiY/rKIqKIXyvemBFHbeijHE25sP7eKltnqcFqAssUE6+sX6vusN9A4luzToOqHQkUZexiCKxvuGagh7JA== - dependencies: - axios "0.24.0" - axios-retry "^3.1.9" - component-type "^1.2.1" - join-component "^1.1.0" - md5 "^2.3.0" - ms "^2.1.3" - remove-trailing-slash "^0.1.1" - uuid "^8.3.2" - posthog-node@4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-4.0.1.tgz#eb8b6cdf68c3fdd0dc2b75e8aab2e0ec3727fb2a" @@ -19494,11 +19433,6 @@ remixicon@2.5.0: resolved "https://registry.yarnpkg.com/remixicon/-/remixicon-2.5.0.tgz#b5e245894a1550aa23793f95daceadbf96ad1a41" integrity sha512-q54ra2QutYDZpuSnFjmeagmEiN9IMo56/zz5dDNitzKD23oFRw77cWo4TsrAdmdkPiEn8mxlrTqxnkujDbEGww== -remove-trailing-slash@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz#be2285a59f39c74d1bce4f825950061915e3780d" - integrity sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA== - request@^2.88.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -20786,16 +20720,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20886,7 +20811,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20900,13 +20825,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22862,7 +22780,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22880,15 +22798,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From fc9b54cb858c743f0530947b7aa984e974e6ceb9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 13:01:33 +0100 Subject: [PATCH 052/195] Mostly solving type errors around passing the view all the way down, got a fair few left. --- .../src/api/controllers/row/external.ts | 21 +++--- .../server/src/api/controllers/row/index.ts | 9 +-- .../src/api/controllers/row/utils/utils.ts | 16 ++--- packages/server/src/sdk/app/rows/external.ts | 65 +++++++++++++------ packages/server/src/sdk/app/rows/rows.ts | 13 ++-- .../src/sdk/app/rows/search/external.ts | 13 ++-- .../server/src/sdk/app/rows/search/utils.ts | 1 - packages/server/src/sdk/app/rows/utils.ts | 8 +-- .../src/utilities/rowProcessor/index.ts | 13 ++-- 9 files changed, 89 insertions(+), 70 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index fb992059f4..2d4012dfcf 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -42,6 +42,7 @@ export async function handleRequest( export async function patch(ctx: UserCtx) { const source = await utils.getSource(ctx) + const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body const { row: dataToUpdate } = await inputProcessing( @@ -58,11 +59,11 @@ export async function patch(ctx: UserCtx) { throw { validation: validateResult.errors } } - const beforeRow = await sdk.rows.external.getRow(tableId, _id, { + const beforeRow = await sdk.rows.external.getRow(table._id!, _id, { relationships: true, }) - const response = await handleRequest(Operation.UPDATE, tableId, { + const response = await handleRequest(Operation.UPDATE, source, { id: breakRowIdField(_id), row: dataToUpdate, }) @@ -70,7 +71,7 @@ export async function patch(ctx: UserCtx) { // The id might have been changed, so the refetching would fail. Recalculating the id just in case const updatedId = generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id - const row = await sdk.rows.external.getRow(tableId, updatedId, { + const row = await sdk.rows.external.getRow(table._id!, updatedId, { relationships: true, }) @@ -78,7 +79,6 @@ export async function patch(ctx: UserCtx) { outputProcessing(table, row, { squash: true, preserveLinks: true, - fromViewId: viewId, }), outputProcessing(table, beforeRow, { squash: true, @@ -95,9 +95,9 @@ export async function patch(ctx: UserCtx) { } export async function destroy(ctx: UserCtx) { - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) const _id = ctx.request.body._id - const { row } = await handleRequest(Operation.DELETE, tableId, { + const { row } = await handleRequest(Operation.DELETE, source, { id: breakRowIdField(_id), includeSqlRelationships: IncludeRelationship.EXCLUDE, }) @@ -106,11 +106,11 @@ export async function destroy(ctx: UserCtx) { export async function bulkDestroy(ctx: UserCtx) { const { rows } = ctx.request.body - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) let promises: Promise<{ row: Row; table: Table }>[] = [] for (let row of rows) { promises.push( - handleRequest(Operation.DELETE, tableId, { + handleRequest(Operation.DELETE, source, { id: breakRowIdField(row._id), includeSqlRelationships: IncludeRelationship.EXCLUDE, }) @@ -125,6 +125,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId + const source = await utils.getSource(ctx) const { tableId } = utils.getSourceId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) const datasource: Datasource = await sdk.datasources.get(datasourceId) @@ -132,7 +133,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { ctx.throw(400, "Datasource has not been configured for plus API.") } const tables = datasource.entities - const response = await handleRequest(Operation.READ, tableId, { + const response = await handleRequest(Operation.READ, source, { id, datasource, includeSqlRelationships: IncludeRelationship.INCLUDE, @@ -156,7 +157,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTable, { tables, filters: { oneOf: { diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 2e5785157d..ac2c09e7a6 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -221,7 +221,7 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - tableId, + sourceId: tableId, } ctx.status = 200 @@ -229,14 +229,15 @@ export async function search(ctx: Ctx) { } export async function validate(ctx: Ctx) { - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) + const table = await utils.getTableFromSource(source) // external tables are hard to validate currently - if (isExternalTableID(tableId)) { + if (isExternalTableID(table._id!)) { ctx.body = { valid: true, errors: {} } } else { ctx.body = await sdk.rows.utils.validate({ row: ctx.request.body, - tableId, + source, }) } } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 91c0fc966f..e5397ed4a5 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -106,19 +106,11 @@ export async function getSource(ctx: Ctx): Promise
{ return sdk.tables.getTable(tableId) } -export async function validate( - opts: { row: Row } & ({ tableId: string } | { table: Table }) -) { - let fetchedTable: Table - if ("tableId" in opts) { - fetchedTable = await sdk.tables.getTable(opts.tableId) - } else { - fetchedTable = opts.table +export async function getTableFromSource(source: Table | ViewV2) { + if (sdk.views.isView(source)) { + return await sdk.views.getTable(source.id) } - return sdk.rows.utils.validate({ - ...opts, - table: fetchedTable, - }) + return source } function fixBooleanFields({ row, table }: { row: Row; table: Table }) { diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 7630e5638b..24ec2302e7 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -1,5 +1,11 @@ -import { IncludeRelationship, Operation, Row } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { + IncludeRelationship, + Operation, + Row, + Table, + ViewV2, +} from "@budibase/types" +import { docIds, HTTPError } from "@budibase/backend-core" import { handleRequest } from "../../../api/controllers/row/external" import { breakRowIdField } from "../../../integrations/utils" import sdk from "../../../sdk" @@ -12,11 +18,21 @@ import isEqual from "lodash/fp/isEqual" import { tryExtractingTableAndViewId } from "./utils" export async function getRow( - tableId: string, + sourceId: string | Table | ViewV2, rowId: string, opts?: { relationships?: boolean } ) { - const response = await handleRequest(Operation.READ, tableId, { + let source: Table | ViewV2 + if (typeof sourceId === "string") { + if (docIds.isViewId(sourceId)) { + source = await sdk.views.get(sourceId) + } else { + source = await sdk.tables.getTable(sourceId) + } + } else { + source = sourceId + } + const response = await handleRequest(Operation.READ, source, { id: breakRowIdField(rowId), includeSqlRelationships: opts?.relationships ? IncludeRelationship.INCLUDE @@ -27,45 +43,50 @@ export async function getRow( } export async function save( - tableOrViewId: string, + sourceId: string, inputs: Row, userId: string | undefined ) { - const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - const table = await sdk.tables.getTable(tableId) + const { tableId, viewId } = tryExtractingTableAndViewId(sourceId) + let source: Table | ViewV2 + if (viewId) { + source = await sdk.views.get(viewId) + } else { + source = await sdk.tables.getTable(tableId) + } + const { table: updatedTable, row } = await inputProcessing( userId, - cloneDeep(table), + cloneDeep(source), inputs ) const validateResult = await sdk.rows.utils.validate({ row, - tableId, + source, }) if (!validateResult.valid) { throw { validation: validateResult.errors } } - const response = await handleRequest(Operation.CREATE, tableId, { + const response = await handleRequest(Operation.CREATE, source, { row, }) - if (!isEqual(table, updatedTable)) { + if (sdk.tables.isTable(source) && !isEqual(source, updatedTable)) { await sdk.tables.saveTable(updatedTable) } const rowId = response.row._id if (rowId) { - const row = await getRow(tableId, rowId, { + const row = await getRow(source, rowId, { relationships: true, }) return { ...response, - row: await outputProcessing(table, row, { + row: await outputProcessing(source, row, { preserveLinks: true, squash: true, - fromViewId: viewId, }), } } else { @@ -76,7 +97,14 @@ export async function save( export async function find(tableOrViewId: string, rowId: string): Promise { const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - const row = await getRow(tableId, rowId, { + let source: Table | ViewV2 + if (viewId) { + source = await sdk.views.get(viewId) + } else { + source = await sdk.tables.getTable(tableId) + } + + const row = await getRow(source, rowId, { relationships: true, }) @@ -84,11 +112,10 @@ export async function find(tableOrViewId: string, rowId: string): Promise { throw new HTTPError("Row not found", 404) } - const table = await sdk.tables.getTable(tableId) - // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case - return await outputProcessing(table, row, { + // Preserving links, as the outputProcessing does not support external rows + // yet and we don't need it in this use case + return await outputProcessing(source, row, { squash: true, preserveLinks: true, - fromViewId: viewId, }) } diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index ef25d06baf..25bdf1fd4f 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,9 +1,6 @@ import { db as dbCore, context, docIds } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" -import { - extractViewInfoFromID, - getRowParams, -} from "../../../db/utils" +import { extractViewInfoFromID, getRowParams } from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" @@ -36,13 +33,13 @@ function pickApi(tableOrViewId: string) { } export async function save( - tableOrViewId: string, + sourceId: string, row: Row, userId: string | undefined ) { - return pickApi(tableOrViewId).save(tableOrViewId, row, userId) + return pickApi(sourceId).save(sourceId, row, userId) } -export async function find(tableOrViewId: string, rowId: string) { - return pickApi(tableOrViewId).find(tableOrViewId, rowId) +export async function find(sourceId: string, rowId: string) { + return pickApi(sourceId).find(sourceId, rowId) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 5584b3f110..a41ae8dcda 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -106,9 +106,9 @@ export async function search( includeSqlRelationships: IncludeRelationship.INCLUDE, } const [{ rows, rawResponseSize }, totalRows] = await Promise.all([ - handleRequest(Operation.READ, tableId, parameters), + handleRequest(Operation.READ, source, parameters), countRows - ? handleRequest(Operation.COUNT, tableId, parameters) + ? handleRequest(Operation.COUNT, source, parameters) : Promise.resolve(undefined), ]) @@ -200,7 +200,7 @@ export async function exportRows( } let result = await search( - { tableId, query: requestQuery, sort, sortOrder }, + { sourceId: table._id!, query: requestQuery, sort, sortOrder }, table ) let rows: Row[] = [] @@ -256,10 +256,10 @@ export async function exportRows( } export async function fetch(tableId: string): Promise { - const response = await handleRequest(Operation.READ, tableId, { + const table = await sdk.tables.getTable(tableId) + const response = await handleRequest(Operation.READ, table, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) - const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response.rows, { preserveLinks: true, squash: true, @@ -267,7 +267,8 @@ export async function fetch(tableId: string): Promise { } export async function fetchRaw(tableId: string): Promise { - const response = await handleRequest(Operation.READ, tableId, { + const table = await sdk.tables.getTable(tableId) + const response = await handleRequest(Operation.READ, table, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) return response.rows diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index d727e58887..6548f963b8 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -9,7 +9,6 @@ import { SearchResponse, Row, RowSearchParams, - ViewV2, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 3899009f13..f17d3e4a03 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -21,6 +21,7 @@ import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds } from "@budibase/backend-core" +import { getTableFromSource } from "../../../api/controllers/row/utils" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -149,12 +150,7 @@ export async function validate({ valid: boolean errors: Record }> { - let table: Table - if (sdk.views.isView(source)) { - table = await sdk.views.getTable(source.id) - } else { - table = source - } + const table = await getTableFromSource(source) const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 73768fdd57..9aa53e18c1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -19,6 +19,7 @@ import { RowAttachment, Table, User, + ViewV2, } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { @@ -34,7 +35,11 @@ import { PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" import { processString } from "@budibase/string-templates" -import { isUserMetadataTable } from "../../api/controllers/row/utils" +import { + getTableFromSource, + isUserMetadataTable, +} from "../../api/controllers/row/utils" +import sdk from "../../sdk" export * from "./utils" export * from "./attachments" @@ -170,11 +175,12 @@ export function coerce(row: any, type: string) { */ export async function inputProcessing( userId: string | null | undefined, - table: Table, + source: Table | ViewV2, row: Row, opts?: AutoColumnProcessingOpts ) { const clonedRow = cloneDeep(row) + const table = await getTableFromSource(source) const dontCleanseKeys = ["type", "_id", "_rev", "tableId"] for (const [key, value] of Object.entries(clonedRow)) { @@ -243,14 +249,13 @@ export async function inputProcessing( * @returns the enriched rows will be returned. */ export async function outputProcessing( - table: Table, + source: Table | ViewV2, rows: T, opts: { squash?: boolean preserveLinks?: boolean fromRow?: Row skipBBReferences?: boolean - fromViewId?: string aggregations?: Aggregation[] } = { squash: true, From c643c82654e4feb9987ff70e8facdd735d849c16 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 13:44:26 +0100 Subject: [PATCH 053/195] Fix for SQL server. --- packages/backend-core/src/sql/sql.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4bdec363a4..2b20938981 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1031,7 +1031,9 @@ class InternalBuilder { .select(`${fromAlias}.*`) // @ts-ignore - from alias syntax not TS supported .from({ - [fromAlias]: subQuery.select(`${toAlias}.*`), + [fromAlias]: subQuery + .select(`${toAlias}.*`) + .limit(getRelationshipLimit()), })} FOR JSON PATH))` ) break From 2337a4b0b3e6916c8e7c3ce4a41f79b3507c3f68 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 24 Sep 2024 13:29:21 +0000 Subject: [PATCH 054/195] Bump version to 2.32.7 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d695869907..272a1dd0c6 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.6", + "version": "2.32.7", "npmClient": "yarn", "packages": [ "packages/*", From 0c6946af62fae3a1f17926fd7f4c2cbed72486c4 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 15:01:05 +0100 Subject: [PATCH 055/195] more automation tests --- .../tests/scenarios/scenarios.spec.ts | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index b54b7c0e0e..4e9340992d 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -196,6 +196,91 @@ describe("Automation Scenarios", () => { ) }) }) + + it("should trigger an automation which creates and then updates a row", async () => { + const table = await config.createTable({ + name: "TestTable", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + value: { + name: "value", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const builder = createAutomationBuilder({ + name: "Test Create and Update Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .createRow( + { + row: { + name: "Initial Row", + value: 1, + tableId: table._id, + }, + }, + { stepName: "CreateRowStep" } + ) + .updateRow( + { + rowId: "{{ steps.CreateRowStep.row._id }}", + row: { + name: "Updated Row", + value: 2, + tableId: table._id, + }, + meta: {}, + }, + { stepName: "UpdateRowStep" } + ) + .queryRows( + { + tableId: table._id!, + }, + { stepName: "QueryRowsStep" } + ) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[0].outputs).toMatchObject({ + success: true, + row: { + name: "Initial Row", + value: 1, + }, + }) + + expect(results.steps[1].outputs).toMatchObject({ + success: true, + row: { + name: "Updated Row", + value: 2, + }, + }) + + const expectedRows = [{ name: "Updated Row", value: 2 }] + + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining( + expectedRows.map(row => expect.objectContaining(row)) + ) + ) + }) }) describe("Name Based Automations", () => { From 6d8921978b7bc5f5c596d8666b49c0b198bfc930 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:30:39 +0100 Subject: [PATCH 056/195] Quick temporary fix for issue - previously the invalid state was never reset, which can cause apps to go missing completely, updating this so it will fix it self after a short while. --- .../backend-core/src/cache/appMetadata.ts | 6 ++--- .../backend-core/src/db/couch/DatabaseImpl.ts | 25 +++++++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index d442511fb8..144836029f 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -11,6 +11,7 @@ export interface DeletedApp { } const EXPIRY_SECONDS = 3600 +const INVALID_EXPIRY_SECONDS = 60 /** * The default populate app metadata function @@ -48,9 +49,8 @@ export async function getAppMetadata(appId: string): Promise { // app DB left around, but no metadata, it is invalid if (err && err.status === 404) { metadata = { state: AppState.INVALID } - // don't expire the reference to an invalid app, it'll only be - // updated if a metadata doc actually gets stored (app is remade/reverted) - expiry = undefined + // expire invalid apps regularly, in-case it was only briefly invalid + expiry = INVALID_EXPIRY_SECONDS } else { throw err } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index aa4656bf64..180cd22efa 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,6 +43,9 @@ function buildNano(couchInfo: { url: string; cookie: string }) { } type DBCall = () => Promise +type DBCallback = ( + db: Nano.DocumentScope +) => Promise> | DBCall class CouchDBError extends Error implements DBError { status: number @@ -171,8 +174,8 @@ export class DatabaseImpl implements Database { } // this function fetches the DB and handles if DB creation is needed - private async performCall( - call: (db: Nano.DocumentScope) => Promise> | DBCall + private async performCallWithDBCreation( + call: DBCallback ): Promise { const db = this.getDb() const fnc = await call(db) @@ -181,13 +184,24 @@ export class DatabaseImpl implements Database { } catch (err: any) { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() - return await this.performCall(call) + return await this.performCallWithDBCreation(call) } // stripping the error down the props which are safe/useful, drop everything else throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } + private async performCall(call: DBCallback): Promise { + const db = this.getDb() + const fnc = await call(db) + try { + return await fnc() + } catch (err: any) { + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, err) + } + } + async get(id?: string): Promise { return this.performCall(db => { if (!id) { @@ -227,6 +241,7 @@ export class DatabaseImpl implements Database { } async remove(idOrDoc: string | Document, rev?: string) { + // not a read call - but don't create a DB to delete a document return this.performCall(db => { let _id: string let _rev: string @@ -286,7 +301,7 @@ export class DatabaseImpl implements Database { if (!document._id) { throw new Error("Cannot store document without _id field.") } - return this.performCall(async db => { + return this.performCallWithDBCreation(async db => { if (!document.createdAt) { document.createdAt = new Date().toISOString() } @@ -309,7 +324,7 @@ export class DatabaseImpl implements Database { async bulkDocs(documents: AnyDocument[]) { const now = new Date().toISOString() - return this.performCall(db => { + return this.performCallWithDBCreation(db => { return () => db.bulk({ docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })), From ad60f8a8119103478cc46f992e8edee3182b813d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:47:07 +0100 Subject: [PATCH 057/195] All docs returns no docs if 404. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 180cd22efa..650f5ca8b6 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -336,7 +336,21 @@ export class DatabaseImpl implements Database { params: DatabaseQueryOpts ): Promise> { return this.performCall(db => { - return () => db.list(params) + return async () => { + try { + return (await db.list(params)) as AllDocsResponse + } catch (err: any) { + if (err.status === 404) { + return { + offset: 0, + total_rows: 0, + rows: [], + } + } else { + throw err + } + } + } }) } From a5cb1b39dcc7605c962c590fea2d8d8989f3d62d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:51:41 +0100 Subject: [PATCH 058/195] All docs no error fix. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 650f5ca8b6..274c1b9e93 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -340,7 +340,7 @@ export class DatabaseImpl implements Database { try { return (await db.list(params)) as AllDocsResponse } catch (err: any) { - if (err.status === 404) { + if (err.reason === DATABASE_NOT_FOUND) { return { offset: 0, total_rows: 0, From 4a55021844c163c82d98c087a4ddf1fa2dea33db Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 24 Sep 2024 16:15:07 +0100 Subject: [PATCH 059/195] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 7899d07904..26903524ff 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd +Subproject commit 26903524ffb6e8f2d99b667ce7a84f09ea238073 diff --git a/packages/pro b/packages/pro index ec1d2bda75..e2fe0f9cc8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c +Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 From 0eb90cfbea9e8dc16ecc917c1994382fdb901000 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 16:35:53 +0100 Subject: [PATCH 060/195] Type checks pass, now to find out how much stuff I've broken. --- packages/backend-core/src/db/lucene.ts | 16 +++---- .../backend-core/src/db/tests/lucene.spec.ts | 4 +- .../src/api/controllers/row/external.ts | 2 +- .../src/api/controllers/row/internal.ts | 31 +++++------- .../src/api/controllers/row/staticFormula.ts | 32 +++++-------- .../src/api/controllers/row/utils/utils.ts | 18 +------ .../src/api/controllers/table/external.ts | 5 +- .../server/src/api/controllers/table/utils.ts | 3 +- .../src/api/routes/tests/search.spec.ts | 12 ++--- .../src/api/routes/tests/templates.spec.ts | 2 +- packages/server/src/db/linkedRows/index.ts | 26 +++++----- .../integrations/tests/googlesheets.spec.ts | 22 ++++----- packages/server/src/sdk/app/rows/external.ts | 11 +---- packages/server/src/sdk/app/rows/internal.ts | 47 ++++++++++--------- .../sdk/app/rows/search/internal/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 20 +++++--- .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++- .../sdk/app/rows/search/tests/search.spec.ts | 10 ++-- .../sdk/app/rows/search/tests/utils.spec.ts | 12 ++--- .../src/sdk/app/rows/tests/utils.spec.ts | 42 ++++++++--------- packages/server/src/sdk/app/rows/utils.ts | 7 +++ packages/server/src/sdk/app/views/index.ts | 3 +- .../src/utilities/rowProcessor/index.ts | 23 +++++++-- .../tests/inputProcessing.spec.ts | 10 ++-- 24 files changed, 183 insertions(+), 188 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f5ad7e6433..b17c3ddf0d 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -79,8 +79,8 @@ export class QueryBuilder { return this } - setTable(tableId: string) { - this.#query.equal!.tableId = tableId + setSource(sourceId: string) { + this.#query.equal!.tableId = sourceId return this } @@ -638,8 +638,8 @@ async function recursiveSearch( .setSortOrder(params.sortOrder) .setSortType(params.sortType) - if (params.tableId) { - queryBuilder.setTable(params.tableId) + if (params.sourceId) { + queryBuilder.setSource(params.sourceId) } const page = await queryBuilder.run() @@ -672,8 +672,8 @@ export async function paginatedSearch( if (params.version) { search.setVersion(params.version) } - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } if (params.sort) { search @@ -695,8 +695,8 @@ export async function paginatedSearch( // Try fetching 1 row in the next page to see if another page of results // exists or not search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } const nextResults = await search.run() diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index c41bdf88d1..8747f56a4b 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -366,7 +366,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, limit: 1, sort: "property", sortType: SortType.STRING, @@ -390,7 +390,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, query: {}, } ) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 2d4012dfcf..11b6559896 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -45,7 +45,7 @@ export async function patch(ctx: UserCtx) { const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body - const { row: dataToUpdate } = await inputProcessing( + const dataToUpdate = await inputProcessing( ctx.user?._id, cloneDeep(source), rowData diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 33e3c7707b..f5cb42f81d 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -21,18 +21,19 @@ import { import sdk from "../../../sdk" import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" +import { findRow } from "../../../sdk/app/rows/internal" export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow - const dbTable = await sdk.tables.getTable(tableId) try { - oldRow = await outputProcessing( - dbTable, - await utils.findRow(tableId, inputs._id!) - ) + oldRow = await outputProcessing(source, await findRow(tableId, inputs._id!)) } catch (err) { if (isUserTable) { // don't include the rev, it'll be the global rev @@ -48,22 +49,18 @@ export async function patch(ctx: UserCtx) { // need to build up full patch fields before coerce let combinedRow: any = cloneDeep(oldRow) for (let key of Object.keys(inputs)) { - if (!dbTable.schema[key]) continue + if (!table.schema[key]) continue combinedRow[key] = inputs[key] } // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const tableClone = cloneDeep(table) // this returns the table and row incase they have been updated - let { table, row } = await inputProcessing( - ctx.user?._id, - tableClone, - combinedRow - ) + let row = await inputProcessing(ctx.user?._id, tableClone, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -87,10 +84,8 @@ export async function patch(ctx: UserCtx) { return { row: ctx.body as Row, table, oldRow } } - const result = await finaliseRow(table, row, { - oldTable: dbTable, + const result = await finaliseRow(source, row, { updateFormula: true, - fromViewId: viewId, }) return { ...result, oldRow } @@ -186,7 +181,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { sdk.tables.getTable(tableId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - let row = await utils.findRow(tableId, rowId) + let row = await findRow(tableId, rowId) row = await outputProcessing(table, row) const linkVals = links as LinkDocumentValue[] diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 777379db14..386dee7b4a 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -4,10 +4,11 @@ import { processFormulas, } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, FormulaType, FieldType } from "@budibase/types" +import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" import isEqual from "lodash/isEqual" import { cloneDeep } from "lodash/fp" +import sdk from "../../../sdk" /** * This function runs through a list of enriched rows, looks at the rows which @@ -121,33 +122,26 @@ export async function updateAllFormulasInTable(table: Table) { * expects the row to be totally enriched/contain all relationships. */ export async function finaliseRow( - table: Table, + source: Table | ViewV2, row: Row, - { - oldTable, - updateFormula, - fromViewId, - }: { oldTable?: Table; updateFormula: boolean; fromViewId?: string } = { - updateFormula: true, - } + opts?: { updateFormula: boolean } ) { const db = context.getAppDB() + const { updateFormula = true } = opts || {} + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source + row.type = "row" // process the row before return, to include relationships - let enrichedRow = (await outputProcessing(table, cloneDeep(row), { + let enrichedRow = await outputProcessing(source, cloneDeep(row), { squash: false, - })) as Row + }) // use enriched row to generate formulas for saving, specifically only use as context row = await processFormulas(table, row, { dynamic: false, contextRows: [enrichedRow], }) - // don't worry about rev, tables handle rev/lastID updates - // if another row has been written since processing this will - // handle the auto ID clash - if (oldTable && !isEqual(oldTable, table)) { - await db.put(table) - } const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev @@ -158,8 +152,6 @@ export async function finaliseRow( if (updateFormula) { await updateRelatedFormula(table, enrichedRow) } - const squashed = await linkRows.squashLinks(table, enrichedRow, { - fromViewId, - }) + const squashed = await linkRows.squashLinks(source, enrichedRow) return { row: enrichedRow, squashed, table } } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index e5397ed4a5..0f565b6951 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context, docIds } from "@budibase/backend-core" +import { docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -20,7 +20,6 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" -import { getFullUser } from "../../../../utilities/users" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -60,21 +59,6 @@ export async function processRelationshipFields( return row } -export async function findRow(tableId: string, rowId: string) { - const db = context.getAppDB() - let row: Row - // TODO remove special user case in future - if (tableId === utils.InternalTables.USER_METADATA) { - row = await getFullUser(rowId) - } else { - row = await db.get(rowId) - } - if (row.tableId !== tableId) { - throw "Supplied tableId does not match the rows tableId" - } - return row -} - export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index c3356919c8..5b15d3d9c7 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -113,11 +113,10 @@ export async function bulkImport( const processed = await inputProcessing(ctx.user?._id, table, row, { noAutoRelationships: true, }) - parsedRows.push(processed.row) - table = processed.table + parsedRows.push(processed) } - await handleRequest(Operation.BULK_UPSERT, table._id!, { + await handleRequest(Operation.BULK_UPSERT, table, { rows: parsedRows, }) await events.rows.imported(table, parsedRows.length) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 269f079ae8..d568e5f33e 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -139,8 +139,7 @@ export async function importToRows( const processed = await inputProcessing(user?._id, table, row, { noAutoRelationships: true, }) - row = processed.row - table = processed.table + row = processed // However here we must reference the original table, as we want to mutate // the real schema of the table passed in, not the clone used for diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c770c4e460..5788d9195a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -157,7 +157,7 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), this.query) } else { - return config.api.row.search(this.query.tableId, this.query) + return config.api.row.search(this.query.sourceId, this.query) } } @@ -327,8 +327,8 @@ describe.each([ } } - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: table._id! }) + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, sourceId: table._id! }) } function expectQuery(query: SearchFilters) { @@ -1898,7 +1898,7 @@ describe.each([ let { rows: fullRowList } = await config.api.row.search( table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, } ) @@ -1909,7 +1909,7 @@ describe.each([ rowCount: number = 0 do { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 1, paginate: true, query: {}, @@ -1933,7 +1933,7 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 3, query: {}, bookmark, diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 6f4d468a68..4290b4386f 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -113,7 +113,7 @@ describe("/templates", () => { expect(users.name).toBe("Users") const { rows } = await config.api.row.search(agencyProjects._id!, { - tableId: agencyProjects._id!, + sourceId: agencyProjects._id!, query: {}, }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 4222f9b5e4..6e65ab36d1 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -255,31 +255,31 @@ export type SquashTableFields = Record * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( - table: Table, - enriched: T, - options?: { - fromViewId?: string - } + source: Table | ViewV2, + enriched: T ): Promise { const allowRelationshipSchemas = await features.flags.isEnabled( FeatureFlag.ENRICHED_RELATIONSHIPS ) let viewSchema: Record = {} - if (options?.fromViewId) { - const view = Object.values(table.views || {}).find( - (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId - ) - - if (view && helpers.views.isCalculationView(view)) { + if (sdk.views.isView(source)) { + if (helpers.views.isCalculationView(source)) { return enriched } - if (allowRelationshipSchemas && view) { - viewSchema = view.schema || {} + if (allowRelationshipSchemas) { + viewSchema = source.schema || {} } } + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 34be1c0c6c..91addf8a50 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -219,7 +219,7 @@ describe("Google Sheets Integration", () => { }) let resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -228,7 +228,7 @@ describe("Google Sheets Integration", () => { while (resp.hasNextPage) { resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -637,7 +637,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Foo", @@ -651,7 +651,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEqual: { name: "Foo", @@ -666,7 +666,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { empty: { name: null, @@ -679,7 +679,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEmpty: { name: null, @@ -692,7 +692,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with one of filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { name: ["Foo", "Bar"], @@ -707,7 +707,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with fuzzy filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { fuzzy: { name: "oo", @@ -721,7 +721,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with range filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { range: { name: { @@ -750,7 +750,7 @@ describe("Google Sheets Integration", () => { }) let response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, @@ -759,7 +759,7 @@ describe("Google Sheets Integration", () => { while (response.hasNextPage) { response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 24ec2302e7..060ef3738a 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -14,7 +14,6 @@ import { outputProcessing, } from "../../../utilities/rowProcessor" import cloneDeep from "lodash/fp/cloneDeep" -import isEqual from "lodash/fp/isEqual" import { tryExtractingTableAndViewId } from "./utils" export async function getRow( @@ -55,11 +54,7 @@ export async function save( source = await sdk.tables.getTable(tableId) } - const { table: updatedTable, row } = await inputProcessing( - userId, - cloneDeep(source), - inputs - ) + const row = await inputProcessing(userId, cloneDeep(source), inputs) const validateResult = await sdk.rows.utils.validate({ row, @@ -73,10 +68,6 @@ export async function save( row, }) - if (sdk.tables.isTable(source) && !isEqual(source, updatedTable)) { - await sdk.tables.saveTable(updatedTable) - } - const rowId = response.row._id if (rowId) { const row = await getRow(source, rowId, { diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index b5b3437e5e..f51bcc37a4 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -1,5 +1,5 @@ import { context, db } from "@budibase/backend-core" -import { Row } from "@budibase/types" +import { Row, Table, ViewV2 } from "@budibase/types" import sdk from "../../../sdk" import cloneDeep from "lodash/fp/cloneDeep" import { finaliseRow } from "../../../api/controllers/row/staticFormula" @@ -10,7 +10,7 @@ import { import * as linkRows from "../../../db/linkedRows" import { InternalTables } from "../../../db/utils" import { getFullUser } from "../../../utilities/users" -import { tryExtractingTableAndViewId } from "./utils" +import { getSource, tryExtractingTableAndViewId } from "./utils" export async function save( tableOrViewId: string, @@ -20,21 +20,28 @@ export async function save( const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) inputs.tableId = tableId + let source: Table | ViewV2 + let table: Table + if (viewId) { + source = await sdk.views.get(viewId) + table = await sdk.views.getTable(viewId) + } else { + source = await sdk.tables.getTable(tableId) + table = source + } + if (!inputs._rev && !inputs._id) { inputs._id = db.generateRowID(inputs.tableId) } - // this returns the table and row incase they have been updated - const dbTable = await sdk.tables.getTable(inputs.tableId) - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const sourceClone = cloneDeep(source) - let { table, row } = await inputProcessing(userId, tableClone, inputs) + let row = await inputProcessing(userId, sourceClone, inputs) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -49,24 +56,18 @@ export async function save( table, })) as Row - return finaliseRow(table, row, { - oldTable: dbTable, - updateFormula: true, - fromViewId: viewId, + return finaliseRow(table, row, { updateFormula: true }) +} + +export async function find(sourceId: string, rowId: string): Promise { + const source = await getSource(sourceId) + return await outputProcessing(source, await findRow(sourceId, rowId), { + squash: true, }) } -export async function find(tableOrViewId: string, rowId: string): Promise { - const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - - const table = await sdk.tables.getTable(tableId) - let row = await findRow(tableId, rowId) - - row = await outputProcessing(table, row, { squash: true, fromViewId: viewId }) - return row -} - -async function findRow(tableId: string, rowId: string) { +export async function findRow(sourceId: string, rowId: string) { + const { tableId } = tryExtractingTableAndViewId(sourceId) const db = context.getAppDB() let row: Row // TODO remove special user case in future diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index 6617fc376c..c9e2aba237 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -64,7 +64,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { let searchResponse = await sdk.rows.search({ - tableId, + sourceId: tableId, query, sort, sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 2c149e5b21..24a7de1307 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -8,21 +8,30 @@ import { SortType, Table, User, + ViewV2, } from "@budibase/types" import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" +import sdk from "../../../../" export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options + const { sourceId } = options + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } const { paginate, query } = options const params: RowSearchParams = { - tableId: options.tableId, + sourceId: options.sourceId, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, @@ -50,7 +59,7 @@ export async function search( // Enrich search results with relationships if (response.rows && response.rows.length) { // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { + if (sourceId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } @@ -59,9 +68,8 @@ export async function search( response.rows = response.rows.map((r: any) => pick(r, fields)) } - response.rows = await outputProcessing(table, response.rows, { + response.rows = await outputProcessing(source, response.rows, { squash: true, - fromViewId: options.viewId, }) } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index b5bf8e752f..90fb082214 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -15,6 +15,7 @@ import { SortType, SqlClient, Table, + ViewV2, } from "@budibase/types" import { buildInternalRelationships, @@ -292,11 +293,18 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, - table: Table, + source: Table | ViewV2, opts?: { retrying?: boolean } ): Promise> { let { paginate, query, ...params } = cloneDeep(options) + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) // make sure we have the mapped/latest table @@ -406,7 +414,6 @@ export async function search( let finalRows = await outputProcessing(table, processed, { preserveLinks: true, squash: true, - fromViewId: options.viewId, aggregations: options.aggregations, }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index e7fd095865..194f2dd4e3 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -122,7 +122,7 @@ describe.each([ it("querying by fields will always return data attribute columns", async () => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -142,7 +142,7 @@ describe.each([ it("will decode _id in oneOf query", async () => { await config.doInContext(config.appId, async () => { const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], @@ -174,7 +174,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, }) expect(result.rows).toHaveLength(10) @@ -205,7 +205,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -229,7 +229,7 @@ describe.each([ async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: { $or: { conditions: [ diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index 0698f727df..e3f241f15a 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": email, @@ -90,10 +90,8 @@ describe.each([tableWithUserCol, tableWithUsersCol])( }) it("shouldn't error if no query supplied", () => { - const params: any = { - tableId, - } - const output = searchInputMapping(col, params) + // @ts-expect-error - intentionally passing in a bad type + const output = searchInputMapping(col, { sourceId: tableId }) expect(output.query).toBeUndefined() }) } diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 55cdf9ea20..548b2b6bc9 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -33,7 +33,7 @@ describe("validate", () => { it("should accept empty values", async () => { const row = {} const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) expect(output.errors).toEqual({}) }) @@ -43,7 +43,7 @@ describe("validate", () => { time: `${hour()}:${minute()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -52,7 +52,7 @@ describe("validate", () => { time: `${hour()}:${minute()}:${second()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -67,7 +67,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ['"time" is not a valid time'] }) }) @@ -91,7 +91,7 @@ describe("validate", () => { `${generator.integer({ min: 11, max: 23 })}:${minute()}`, ])("should accept values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -100,7 +100,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -125,7 +125,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 12 })}:${minute()}`, ])("should accept values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -134,7 +134,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:16:17"], @@ -156,7 +156,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -166,7 +166,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -178,7 +178,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -199,7 +199,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -208,7 +208,7 @@ describe("validate", () => { "should reject values out range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 10:00"], @@ -226,7 +226,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -237,7 +237,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -257,7 +257,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -267,7 +267,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -279,7 +279,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -301,7 +301,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -311,7 +311,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -323,7 +323,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index f17d3e4a03..d5c0560d9b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -321,3 +321,10 @@ export function tryExtractingTableAndViewId(tableOrViewId: string) { return { tableId: tableOrViewId } } + +export function getSource(tableOrViewId: string) { + if (docIds.isViewId(tableOrViewId)) { + return sdk.views.get(tableOrViewId) + } + return sdk.tables.getTable(tableOrViewId) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index f911385dc1..4251383712 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -40,7 +40,8 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } -export async function getTable(viewId: string): Promise
{ +export async function getTable(view: string | ViewV2): Promise
{ + const viewId = typeof view === "string" ? view : view.id const cached = context.getTableForView(viewId) if (cached) { return cached diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 9aa53e18c1..24c8d11bd1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -73,6 +73,7 @@ export async function processAutoColumn( // check its not user table, or whether any of the processing options have been disabled const shouldUpdateUserFields = !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser + let tableMutated = false for (let [key, schema] of Object.entries(table.schema)) { if (!schema.autocolumn) { continue @@ -105,10 +106,17 @@ export async function processAutoColumn( row[key] = schema.lastID + 1 schema.lastID++ table.schema[key] = schema + tableMutated = true } break } } + + if (tableMutated) { + const db = context.getAppDB() + const resp = await db.put(table) + table._rev = resp.rev + } } async function processDefaultValues(table: Table, row: Row) { @@ -235,8 +243,7 @@ export async function inputProcessing( await processAutoColumn(userId, table, clonedRow, opts) await processDefaultValues(table, clonedRow) - - return { table, row: clonedRow } + return clonedRow } /** @@ -271,6 +278,14 @@ export async function outputProcessing( } else { safeRows = rows } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // attach any linked row information let enriched = !opts.preserveLinks ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { @@ -360,9 +375,7 @@ export async function outputProcessing( enriched = await processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { - enriched = await linkRows.squashLinks(table, enriched, { - fromViewId: opts?.fromViewId, - }) + enriched = await linkRows.squashLinks(source, enriched) } // remove null properties to match internal API diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 244ea3794c..1a75cd6830 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -65,7 +65,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferenceMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledTimes( 1 @@ -117,7 +117,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferencesMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).toHaveBeenCalledTimes( 1 @@ -164,7 +164,7 @@ describe("rowProcessor - inputProcessing", () => { name: "Jack", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({ ...newRow, user: undefined }) @@ -207,7 +207,7 @@ describe("rowProcessor - inputProcessing", () => { user: userValue, } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) if (userValue === undefined) { // The 'user' field is omitted @@ -262,7 +262,7 @@ describe("rowProcessor - inputProcessing", () => { user: "123", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({ From d7d8284caf06b512b915c58c1e5d52cd56ebf6bc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:42:25 +0200 Subject: [PATCH 061/195] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c4a39ae8a9..c34f4fb3ac 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,6 +22,7 @@ import { TableSchema, ViewFieldMetadata, RenameColumn, + FeatureFlag, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2213,6 +2214,100 @@ describe.each([ }) ) }) + + describe("foreign relationship columns", () => { + const createMainTable = async ( + links: { + name: string + tableId: string + fk: string + }[] + ) => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { title: { name: "title", type: FieldType.STRING } }, + }) + ) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + ...links.reduce((acc, c) => { + acc[c.name] = { + name: c.name, + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: c.tableId, + fieldName: c.fk, + constraints: { type: "array" }, + } + return acc + }, {}), + }, + }) + return table + } + const createAuxTable = (schema: TableSchema) => + config.api.table.save( + saveTableRequest({ + primaryDisplay: "name", + schema: { + ...schema, + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + + it("returns squashed fields respecting the view config", async () => { + const auxTable = await createAuxTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + age: generator.age(), + }) + const row = await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: false, readonly: false }, + age: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + age: auxRow.age, + }, + ], + }), + ]) + }) + }) }) describe("permissions", () => { From 53b4634cffb48d254c7d4e1b5a91f4a61f017880 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:51:05 +0200 Subject: [PATCH 062/195] Add enrichment tests --- .../src/api/routes/tests/viewV2.spec.ts | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c34f4fb3ac..a2af95bee3 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -23,6 +23,7 @@ import { ViewFieldMetadata, RenameColumn, FeatureFlag, + BBReferenceFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2270,7 +2271,7 @@ describe.each([ name: generator.name(), age: generator.age(), }) - const row = await config.api.row.save(table._id!, { + await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], }) @@ -2307,6 +2308,69 @@ describe.each([ }), ]) }) + + it("enriches squashed fields", async () => { + const auxTable = await createAuxTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + constraints: { presence: true }, + }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const user = config.getUser() + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + user: user._id, + }) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: true, readonly: true }, + user: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + name: auxRow.name, + user: { + _id: user._id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + primaryDisplay: user.email, + }, + }, + ], + }), + ]) + }) }) }) From 7a7ce3dc629ed365a67cbdbae4d812d80ffc7440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:56:07 +0200 Subject: [PATCH 063/195] Fix --- packages/server/src/db/linkedRows/index.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..c2b043785f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -10,7 +10,7 @@ import flatten from "lodash/flatten" import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" -import { processFormulas } from "../../utilities/rowProcessor" +import { outputProcessing, processFormulas } from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" import { ContextUser, @@ -275,7 +275,7 @@ export async function squashLinks( // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) - const enrichedArray = !isArray ? [enriched] : enriched + const enrichedArray = !isArray ? [enriched as Row] : (enriched as Row[]) for (const row of enrichedArray) { // this only fetches the table if its not already in array const rowTable = await getLinkedTable(row.tableId!, linkedTables) @@ -292,6 +292,9 @@ export async function squashLinks( obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) if (viewSchema[column]?.columns) { + const enrichedLink = await outputProcessing(linkedTable, link, { + squash: false, + }) const squashFields = Object.entries(viewSchema[column].columns) .filter(([columnName, viewColumnConfig]) => { const tableColumn = linkedTable.schema[columnName] @@ -312,7 +315,7 @@ export async function squashLinks( .map(([columnName]) => columnName) for (const relField of squashFields) { - obj[relField] = link[relField] + obj[relField] = enrichedLink[relField] } } @@ -321,5 +324,5 @@ export async function squashLinks( row[column] = newLinks } } - return isArray ? enrichedArray : enrichedArray[0] + return (isArray ? enrichedArray : enrichedArray[0]) as T } From e3256cb005ec6f8517e5db995f8c555ba00f7b38 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 17:46:38 +0100 Subject: [PATCH 064/195] Fix row.spec.ts. --- packages/backend-core/src/db/lucene.ts | 1 - .../src/api/controllers/row/external.ts | 4 ++-- .../server/src/api/controllers/row/index.ts | 4 ++-- .../src/api/controllers/row/utils/utils.ts | 8 +++++++- .../server/src/api/controllers/table/index.ts | 9 ++------- .../server/src/api/routes/tests/row.spec.ts | 12 ++++++++--- packages/server/src/db/linkedRows/index.ts | 7 ++++--- packages/server/src/sdk/app/rows/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 2 +- .../src/sdk/app/rows/search/internal/sqs.ts | 20 ++++++++----------- 10 files changed, 36 insertions(+), 33 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index b17c3ddf0d..7f58c7068e 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -612,7 +612,6 @@ async function runQuery( * limit {number} The number of results to fetch * bookmark {string|null} Current bookmark in the recursive search * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} */ async function recursiveSearch( dbName: string, diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 11b6559896..18a9be5087 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,11 +76,11 @@ export async function patch(ctx: UserCtx) { }) const [enrichedRow, oldRow] = await Promise.all([ - outputProcessing(table, row, { + outputProcessing(source, row, { squash: true, preserveLinks: true, }), - outputProcessing(table, beforeRow, { + outputProcessing(source, beforeRow, { squash: true, preserveLinks: true, }), diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index ac2c09e7a6..84b5edcc12 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -207,7 +207,7 @@ export async function destroy(ctx: UserCtx) { } export async function search(ctx: Ctx) { - const { tableId } = utils.getSourceId(ctx) + const { tableId, viewId } = utils.getSourceId(ctx) await context.ensureSnippetContext(true) @@ -221,7 +221,7 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - sourceId: tableId, + sourceId: viewId || tableId, } ctx.status = 200 diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 0f565b6951..45abd93930 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -112,7 +112,7 @@ function fixBooleanFields({ row, table }: { row: Row; table: Table }) { export async function sqlOutputProcessing( rows: DatasourcePlusQueryResponse, - table: Table, + source: Table | ViewV2, tables: Record, relationships: RelationshipsJson[], opts?: { sqs?: boolean; aggregations?: Aggregation[] } @@ -120,6 +120,12 @@ export async function sqlOutputProcessing( if (isKnexEmptyReadResponse(rows)) { return [] } + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } let finalRows: { [key: string]: Row } = {} for (let row of rows as Row[]) { let rowId = row._id diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 0e16077092..7bec1581b4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -33,7 +33,7 @@ import { import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" import { builderSocket } from "../../../websockets" -import { cloneDeep, isEqual } from "lodash" +import { cloneDeep } from "lodash" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -149,12 +149,7 @@ export async function bulkImport( ctx: UserCtx ) { const tableId = ctx.params.tableId - let tableBefore = await sdk.tables.getTable(tableId) - let tableAfter = await pickApi({ tableId }).bulkImport(ctx) - - if (!isEqual(tableBefore, tableAfter)) { - await sdk.tables.saveTable(tableAfter) - } + await pickApi({ tableId }).bulkImport(ctx) // right now we don't trigger anything for bulk import because it // can only be done in the builder, but in the future we may need to diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index dc03a21d6d..467faa8e06 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,7 +76,7 @@ async function waitForEvent( } describe.each([ - ["internal", undefined], + ["lucene", undefined], ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], @@ -2453,9 +2453,15 @@ describe.each([ let flagCleanup: (() => void) | undefined beforeAll(async () => { - flagCleanup = setCoreEnv({ + const env = { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`, - }) + } + if (isSqs) { + env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS` + } else { + env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS` + } + flagCleanup = setCoreEnv(env) const aux2Table = await config.api.table.save(saveTableRequest()) const aux2Data = await config.api.row.save(aux2Table._id!, {}) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 6e65ab36d1..bc37fbef42 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -248,10 +248,11 @@ function getPrimaryDisplayValue(row: Row, table?: Table) { export type SquashTableFields = Record /** - * This function will take the given enriched rows and squash the links to only contain the primary display field. - * @param table The table from which the rows originated. + * This function will take the given enriched rows and squash the links to only + * contain the primary display field. + * + * @param source The table or view from which the rows originated. * @param enriched The pre-enriched rows (full docs) which are to be squashed. - * @param squashFields Per link column (key) define which columns are allowed while squashing. * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index f51bcc37a4..13054f43f4 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -56,7 +56,7 @@ export async function save( table, })) as Row - return finaliseRow(table, row, { updateFormula: true }) + return finaliseRow(source, row, { updateFormula: true }) } export async function find(sourceId: string, rowId: string): Promise { diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 24a7de1307..65cf4053d7 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -31,7 +31,7 @@ export async function search( const { paginate, query } = options const params: RowSearchParams = { - sourceId: options.sourceId, + sourceId: table._id!, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 90fb082214..5a9e1ddf24 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -308,8 +308,8 @@ export async function search( const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) // make sure we have the mapped/latest table - if (table?._id) { - table = allTablesMap[table?._id] + if (table._id) { + table = allTablesMap[table._id] } if (!table) { throw new Error("Unable to find table") @@ -322,13 +322,6 @@ export async function search( documentType: DocumentType.ROW, } - let fields = options.fields - if (fields === undefined) { - fields = buildInternalFieldList(table, allTables, { relationships }) - } else { - fields = fields.map(f => mapToUserColumn(f)) - } - if (options.aggregations) { options.aggregations = options.aggregations.map(a => { a.field = mapToUserColumn(a.field) @@ -350,7 +343,10 @@ export async function search( tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, - resource: { fields, aggregations: options.aggregations }, + resource: { + fields: buildInternalFieldList(table, allTables, { relationships }), + aggregations: options.aggregations, + }, relationships, } @@ -394,7 +390,7 @@ export async function search( // make sure JSON columns corrected const processed = builder.convertJsonStringColumns( table, - await sqlOutputProcessing(rows, table, allTablesMap, relationships, { + await sqlOutputProcessing(rows, source, allTablesMap, relationships, { sqs: true, aggregations: options.aggregations, }) @@ -411,7 +407,7 @@ export async function search( } // get the rows - let finalRows = await outputProcessing(table, processed, { + let finalRows = await outputProcessing(source, processed, { preserveLinks: true, squash: true, aggregations: options.aggregations, From f475454bce2a5daa3c745716c853be0a371239c1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 18:07:31 +0100 Subject: [PATCH 065/195] Fix search API break. --- packages/backend-core/src/db/lucene.ts | 16 ++++++------ .../backend-core/src/db/tests/lucene.spec.ts | 4 +-- .../server/src/api/controllers/row/index.ts | 3 ++- .../server/src/api/controllers/row/views.ts | 11 +++----- .../src/api/routes/tests/search.spec.ts | 16 +++++++----- .../src/api/routes/tests/templates.spec.ts | 2 +- .../integrations/tests/googlesheets.spec.ts | 22 ++++++++-------- packages/server/src/sdk/app/rows/search.ts | 25 ++++++++----------- .../src/sdk/app/rows/search/external.ts | 2 +- .../sdk/app/rows/search/internal/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 7 +++--- .../sdk/app/rows/search/tests/search.spec.ts | 10 ++++---- .../sdk/app/rows/search/tests/utils.spec.ts | 8 +++--- packages/types/src/sdk/row.ts | 5 ++-- 14 files changed, 66 insertions(+), 67 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 7f58c7068e..0206bb2140 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -79,8 +79,8 @@ export class QueryBuilder { return this } - setSource(sourceId: string) { - this.#query.equal!.tableId = sourceId + setTable(tableId: string) { + this.#query.equal!.tableId = tableId return this } @@ -637,8 +637,8 @@ async function recursiveSearch( .setSortOrder(params.sortOrder) .setSortType(params.sortType) - if (params.sourceId) { - queryBuilder.setSource(params.sourceId) + if (params.tableId) { + queryBuilder.setTable(params.tableId) } const page = await queryBuilder.run() @@ -671,8 +671,8 @@ export async function paginatedSearch( if (params.version) { search.setVersion(params.version) } - if (params.sourceId) { - search.setSource(params.sourceId) + if (params.tableId) { + search.setTable(params.tableId) } if (params.sort) { search @@ -694,8 +694,8 @@ export async function paginatedSearch( // Try fetching 1 row in the next page to see if another page of results // exists or not search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.sourceId) { - search.setSource(params.sourceId) + if (params.tableId) { + search.setTable(params.tableId) } const nextResults = await search.run() diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 8747f56a4b..c41bdf88d1 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -366,7 +366,7 @@ describe("lucene", () => { }, }, { - sourceId: TABLE_ID, + tableId: TABLE_ID, limit: 1, sort: "property", sortType: SortType.STRING, @@ -390,7 +390,7 @@ describe("lucene", () => { }, }, { - sourceId: TABLE_ID, + tableId: TABLE_ID, query: {}, } ) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 84b5edcc12..680a7671d5 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -221,7 +221,8 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - sourceId: viewId || tableId, + tableId, + viewId, } ctx.status = 200 diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 7008c5e0be..fa75990136 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,8 +3,6 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, - RequiredKeys, - RowSearchParams, SearchFilterKey, LogicalOperator, Aggregation, @@ -83,9 +81,9 @@ export async function searchView( field, })) - const searchOptions: RequiredKeys & - RequiredKeys> = { - sourceId: view.id, + const result = await sdk.rows.search({ + viewId: view.id, + tableId: view.tableId, query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), @@ -94,9 +92,8 @@ export async function searchView( paginate: body.paginate, countRows: body.countRows, aggregations, - } + }) - const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5788d9195a..090514250d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -157,7 +157,11 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), this.query) } else { - return config.api.row.search(this.query.sourceId, this.query) + const sourceId = this.query.viewId || this.query.tableId + if (!sourceId) { + throw new Error("No source ID provided") + } + return config.api.row.search(sourceId, this.query) } } @@ -327,8 +331,8 @@ describe.each([ } } - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, sourceId: table._id! }) + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, tableId: table._id! }) } function expectQuery(query: SearchFilters) { @@ -1898,7 +1902,7 @@ describe.each([ let { rows: fullRowList } = await config.api.row.search( table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, } ) @@ -1909,7 +1913,7 @@ describe.each([ rowCount: number = 0 do { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, limit: 1, paginate: true, query: {}, @@ -1933,7 +1937,7 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, limit: 3, query: {}, bookmark, diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 4290b4386f..6f4d468a68 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -113,7 +113,7 @@ describe("/templates", () => { expect(users.name).toBe("Users") const { rows } = await config.api.row.search(agencyProjects._id!, { - sourceId: agencyProjects._id!, + tableId: agencyProjects._id!, query: {}, }) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 91addf8a50..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -219,7 +219,7 @@ describe("Google Sheets Integration", () => { }) let resp = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, paginate: true, limit: 10, @@ -228,7 +228,7 @@ describe("Google Sheets Integration", () => { while (resp.hasNextPage) { resp = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, paginate: true, limit: 10, @@ -637,7 +637,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with equals filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Foo", @@ -651,7 +651,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not equals filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { notEqual: { name: "Foo", @@ -666,7 +666,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with empty filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { empty: { name: null, @@ -679,7 +679,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not empty filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { notEmpty: { name: null, @@ -692,7 +692,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with one of filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { oneOf: { name: ["Foo", "Bar"], @@ -707,7 +707,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with fuzzy filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { fuzzy: { name: "oo", @@ -721,7 +721,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with range filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { range: { name: { @@ -750,7 +750,7 @@ describe("Google Sheets Integration", () => { }) let response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, @@ -759,7 +759,7 @@ describe("Google Sheets Integration", () => { while (response.hasNextPage) { response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index fa01a6cf13..c1685d8024 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,7 +14,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features, docIds } from "@budibase/backend-core" +import { features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" @@ -38,7 +38,8 @@ export async function search( ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ - sourceId: options.sourceId, + tableId: options.tableId, + viewId: options.viewId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, @@ -76,20 +77,16 @@ export async function search( let source: Table | ViewV2 let table: Table - if (docIds.isTableId(options.sourceId)) { - source = await sdk.tables.getTable(options.sourceId) - table = source - options = searchInputMapping(source, options) - } else if (docIds.isViewId(options.sourceId)) { - source = await sdk.views.get(options.sourceId) - table = await sdk.tables.getTable(source.tableId) + if (options.viewId) { + source = await sdk.views.get(options.viewId) + table = await sdk.views.getTable(source) + options = searchInputMapping(table, options) + } else if (options.tableId) { + source = await sdk.tables.getTable(options.tableId) + table = source options = searchInputMapping(table, options) - - span.addTags({ - tableId: table._id, - }) } else { - throw new Error(`Invalid source ID: ${options.sourceId}`) + throw new Error(`Invalid source ID: ${options.viewId || options.tableId}`) } if (options.query) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index a41ae8dcda..925b472a48 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -200,7 +200,7 @@ export async function exportRows( } let result = await search( - { sourceId: table._id!, query: requestQuery, sort, sortOrder }, + { tableId: table._id!, query: requestQuery, sort, sortOrder }, table ) let rows: Row[] = [] diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index c9e2aba237..6617fc376c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -64,7 +64,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { let searchResponse = await sdk.rows.search({ - sourceId: tableId, + tableId, query, sort, sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 65cf4053d7..694c660625 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -19,8 +19,6 @@ export async function search( options: RowSearchParams, source: Table | ViewV2 ): Promise> { - const { sourceId } = options - let table: Table if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) @@ -31,7 +29,8 @@ export async function search( const { paginate, query } = options const params: RowSearchParams = { - sourceId: table._id!, + tableId: options.tableId, + viewId: options.viewId, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, @@ -59,7 +58,7 @@ export async function search( // Enrich search results with relationships if (response.rows && response.rows.length) { // enrich with global users if from users table - if (sourceId === InternalTables.USER_METADATA) { + if (table._id === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 194f2dd4e3..e7fd095865 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -122,7 +122,7 @@ describe.each([ it("querying by fields will always return data attribute columns", async () => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, fields: ["name", "age"], }) @@ -142,7 +142,7 @@ describe.each([ it("will decode _id in oneOf query", async () => { await config.doInContext(config.appId, async () => { const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: { oneOf: { _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], @@ -174,7 +174,7 @@ describe.each([ }, }) const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, }) expect(result.rows).toHaveLength(10) @@ -205,7 +205,7 @@ describe.each([ }, }) const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, fields: ["name", "age"], }) @@ -229,7 +229,7 @@ describe.each([ async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - sourceId: table._id!, + tableId: table._id!, query: { $or: { conditions: [ diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index e3f241f15a..daa658455b 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { equal: { "1:user": email, @@ -91,7 +91,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't error if no query supplied", () => { // @ts-expect-error - intentionally passing in a bad type - const output = searchInputMapping(col, { sourceId: tableId }) + const output = searchInputMapping(col, { tableId }) expect(output.query).toBeUndefined() }) } diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 2b6ff3a6c6..f81d56c082 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -10,7 +10,8 @@ export interface Aggregation { } export interface SearchParams { - sourceId?: string + tableId?: string + viewId?: string query?: SearchFilters paginate?: boolean bookmark?: string | number @@ -29,7 +30,7 @@ export interface SearchParams { // when searching for rows we want a more extensive search type that requires certain properties export interface RowSearchParams - extends WithRequired {} + extends WithRequired {} export interface SearchResponse { rows: T[] From 9ecb64a99229abc626012be597af55f54eba2e0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 10:32:58 +0200 Subject: [PATCH 066/195] Use sqs flag correctly on test --- packages/server/src/api/routes/tests/viewV2.spec.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a2af95bee3..a47d6dd828 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2291,8 +2291,17 @@ describe.each([ }, }) + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (isLucene) { + flags.push("*:!SQS") + } else if (isSqs) { + flags.push("*:SQS") + } + const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + { + TENANT_FEATURE_FLAGS: flags.join(","), + }, () => config.api.viewV2.search(view.id) ) From 7072244f31a165a0b3af80bae55922b725e6d13a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 12:44:30 +0200 Subject: [PATCH 067/195] Fix --- .../src/api/routes/tests/viewV2.spec.ts | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a47d6dd828..e38e4c2ed5 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -34,6 +34,7 @@ import { roles, withEnv as withCoreEnv, setEnv as setCoreEnv, + env, } from "@budibase/backend-core" import sdk from "../../../sdk" @@ -696,22 +697,23 @@ describe.each([ ) }) - it("cannot update views v1", async () => { - const viewV1 = await config.api.legacyView.save({ - tableId: table._id!, - name: generator.guid(), - filters: [], - schema: {}, - }) + isInternal && + it("cannot update views v1", async () => { + const viewV1 = await config.api.legacyView.save({ + tableId: table._id!, + name: generator.guid(), + filters: [], + schema: {}, + }) - await config.api.viewV2.update(viewV1 as unknown as ViewV2, { - status: 400, - body: { - message: "Only views V2 can be updated", + await config.api.viewV2.update(viewV1 as unknown as ViewV2, { status: 400, - }, + body: { + message: "Only views V2 can be updated", + status: 400, + }, + }) }) - }) it("cannot update the a view with unmatching ids between url and body", async () => { const anotherView = await config.api.viewV2.create({ @@ -2217,6 +2219,21 @@ describe.each([ }) describe("foreign relationship columns", () => { + let envCleanup: () => void + beforeAll(() => { + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (env.TENANT_FEATURE_FLAGS) { + flags.push(...env.TENANT_FEATURE_FLAGS.split(",")) + } + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: flags.join(","), + }) + }) + + afterAll(() => { + envCleanup?.() + }) + const createMainTable = async ( links: { name: string @@ -2263,14 +2280,14 @@ describe.each([ const auxTable = await createAuxTable({ age: { name: "age", type: FieldType.NUMBER }, }) - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - const auxRow = await config.api.row.save(auxTable._id!, { name: generator.name(), age: generator.age(), }) + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], @@ -2291,20 +2308,7 @@ describe.each([ }, }) - const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] - if (isLucene) { - flags.push("*:!SQS") - } else if (isSqs) { - flags.push("*:SQS") - } - - const response = await withCoreEnv( - { - TENANT_FEATURE_FLAGS: flags.join(","), - }, - () => config.api.viewV2.search(view.id) - ) - + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({ aux: [ @@ -2356,10 +2360,7 @@ describe.each([ }, }) - const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, - () => config.api.viewV2.search(view.id) - ) + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({ From 77587c86868b9970daa9664db3a38d3befe6b847 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 12:23:32 +0200 Subject: [PATCH 068/195] Add related columns --- .../GridColumnConfiguration/getColumns.js | 44 ++++++++++++++++--- .../src/components/app/GridBlock.svelte | 1 + .../src/components/grid/stores/datasource.js | 14 ++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 148055d727..384f20cf65 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -1,3 +1,5 @@ +import { FieldType } from "@budibase/types" + const modernize = columns => { if (!columns) { return [] @@ -8,6 +10,7 @@ const modernize = columns => { label: column.displayName, field: column.name, active: true, + related: column.related, })) } @@ -50,12 +53,35 @@ const removeInvalidAddMissing = ( const getDefault = (schema = {}) => { const defaultValues = Object.values(schema) .filter(column => !column.nestedJSON) - .map(column => ({ - label: column.name, - field: column.name, - active: column.visible ?? true, - order: column.visible ? column.order ?? -1 : Number.MAX_SAFE_INTEGER, - })) + .flatMap(column => { + const order = column.visible + ? column.order ?? -1 + : Number.MAX_SAFE_INTEGER + const columns = [ + { + label: column.name, + field: column.name, + active: column.visible ?? true, + order, + }, + ] + + if (column.columns) { + for (const relColumn of Object.keys(column.columns).filter( + relColumn => column.columns[relColumn].visible !== false + )) { + columns.push({ + label: `${relColumn} (${column.name})`, + field: `${column.name}.${relColumn}`, + active: column.visible ?? true, + order, + related: true, + }) + } + } + + return columns + }) defaultValues.sort((a, b) => a.order - b.order) @@ -69,6 +95,7 @@ const toGridFormat = draggableListColumns => { active: entry.active, width: entry.width, conditions: entry.conditions, + related: entry.related, })) } @@ -82,9 +109,12 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { active: column.active, field: column.field, label: column.label, - columnType: schema[column.field].type, + columnType: column.related + ? FieldType.FORMULA + : schema[column.field]?.type, width: column.width, conditions: column.conditions, + related: column.related, }, {} ) diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 30a35b0713..45aba66bab 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -98,6 +98,7 @@ order: idx, conditions: column.conditions, visible: !!column.active, + related: column.related, } if (column.width) { overrides[column.field].width = column.width diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 68053f38ae..3af6948650 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,6 +1,7 @@ import { derived, get } from "svelte/store" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { memo } from "../../../utils" +import { FieldType } from "@budibase/types" export const createStores = () => { const definition = memo(null) @@ -73,6 +74,19 @@ export const deriveStores = context => { } } }) + if ($schemaOverrides) { + Object.keys($schemaOverrides).forEach(field => { + if (!$schemaOverrides[field].related) { + return + } + enrichedSchema[field] = { + ...$schemaOverrides[field], + name: field, + type: FieldType.FORMULA, + related: true, + } + }) + } return enrichedSchema } ) From 55190883763eb2b58f64a10d79fe557c8ee7872a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 12:36:04 +0200 Subject: [PATCH 069/195] Display related --- .../GridColumnConfiguration/getColumns.js | 2 +- .../src/components/grid/stores/datasource.js | 1 - .../src/components/grid/stores/rows.js | 32 +++++++++++++++---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 384f20cf65..419628ead2 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -75,7 +75,7 @@ const getDefault = (schema = {}) => { field: `${column.name}.${relColumn}`, active: column.visible ?? true, order, - related: true, + related: { field: column.name, subField: relColumn }, }) } } diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 3af6948650..6e0b31de4b 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,7 +83,6 @@ export const deriveStores = context => { ...$schemaOverrides[field], name: field, type: FieldType.FORMULA, - related: true, } }) } diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js index fb7a487c8e..e13767cbca 100644 --- a/packages/frontend-core/src/components/grid/stores/rows.js +++ b/packages/frontend-core/src/components/grid/stores/rows.js @@ -6,6 +6,7 @@ import { tick } from "svelte" import { Helpers } from "@budibase/bbui" import { sleep } from "../../../utils/utils" import { FieldType } from "@budibase/types" +import { processStringSync } from "@budibase/string-templates" export const createStores = () => { const rows = writable([]) @@ -42,15 +43,32 @@ export const createStores = () => { } export const deriveStores = context => { - const { rows } = context + const { rows, enrichedSchema } = context // Enrich rows with an index property and any pending changes - const enrichedRows = derived(rows, $rows => { - return $rows.map((row, idx) => ({ - ...row, - __idx: idx, - })) - }) + const enrichedRows = derived( + [rows, enrichedSchema], + ([$rows, $enrichedSchema]) => { + const customColumns = Object.values($enrichedSchema || {}).filter( + f => f.related + ) + return $rows.map((row, idx) => ({ + ...row, + __idx: idx, + ...customColumns.reduce((acc, c) => { + try { + acc[c.name] = processStringSync( + `{{ join (pluck ${c.related.field} '${c.related.subField}') ', ' }}`, + row + ) + } catch { + // It might be some formula not set, or anything being incorrect + } + return acc + }, {}), + })) + } + ) // Generate a lookup map to quick find a row by ID const rowLookupMap = derived(enrichedRows, $enrichedRows => { From 49502afbeed80088fd5e161e4ab6bd886b3cbe9e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:04:57 +0200 Subject: [PATCH 070/195] Default off --- .../settings/controls/GridColumnConfiguration/getColumns.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 419628ead2..22b42dd427 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -73,7 +73,7 @@ const getDefault = (schema = {}) => { columns.push({ label: `${relColumn} (${column.name})`, field: `${column.name}.${relColumn}`, - active: column.visible ?? true, + active: false, order, related: { field: column.name, subField: relColumn }, }) From 8ee6e52f08c133231e94dd6b607830421be4de6a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:24:10 +0200 Subject: [PATCH 071/195] Use types --- .../frontend-core/src/components/grid/stores/datasource.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 6e0b31de4b..ed84dc478a 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -79,10 +79,14 @@ export const deriveStores = context => { if (!$schemaOverrides[field].related) { return } + + const { field: relField, subField: relSubField } = + $schemaOverrides[field].related + enrichedSchema[field] = { ...$schemaOverrides[field], name: field, - type: FieldType.FORMULA, + type: $schema[relField]?.columns?.[relSubField]?.type, } }) } From 574b2e5a3c16668a710df57d2625e30906a6329b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:34:23 +0200 Subject: [PATCH 072/195] Map types --- .../GridColumnConfiguration/getColumns.js | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 22b42dd427..7380e40de0 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -1,5 +1,3 @@ -import { FieldType } from "@budibase/types" - const modernize = columns => { if (!columns) { return [] @@ -20,7 +18,8 @@ const modernize = columns => { const removeInvalidAddMissing = ( columns = [], defaultColumns, - primaryDisplayColumnName + primaryDisplayColumnName, + schema ) => { const defaultColumnNames = defaultColumns.map(column => column.field) const columnNames = columns.map(column => column.field) @@ -47,6 +46,16 @@ const removeInvalidAddMissing = ( combinedColumns[primaryDisplayIndex].active = true } + for (const column of combinedColumns) { + if (!column.related) { + column.columnType = schema[column.field]?.type + continue + } + + const { field: relField, subField: relSubField } = column.related + column.columnType = schema[relField]?.columns?.[relSubField]?.type + } + return combinedColumns } @@ -99,7 +108,7 @@ const toGridFormat = draggableListColumns => { })) } -const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { +const toDraggableListFormat = (gridFormatColumns, createComponent) => { return gridFormatColumns.map(column => { return createComponent( "@budibase/standard-components/labelfield", @@ -109,9 +118,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => { active: column.active, field: column.field, label: column.label, - columnType: column.related - ? FieldType.FORMULA - : schema[column.field]?.type, + columnType: column.columnType, width: column.width, conditions: column.conditions, related: column.related, @@ -131,13 +138,10 @@ const getColumns = ({ const validatedColumns = removeInvalidAddMissing( modernize(columns), getDefault(schema), - primaryDisplayColumnName - ) - const draggableList = toDraggableListFormat( - validatedColumns, - createComponent, + primaryDisplayColumnName, schema ) + const draggableList = toDraggableListFormat(validatedColumns, createComponent) const primary = draggableList .filter(entry => entry.field === primaryDisplayColumnName) .map(instance => ({ ...instance, schema }))[0] From d799bfacb60022c44c95ace62ca24f08fd9dfd04 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:35:11 +0200 Subject: [PATCH 073/195] Readonly --- packages/frontend-core/src/components/grid/stores/datasource.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index ed84dc478a..399ec341ee 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -87,6 +87,7 @@ export const deriveStores = context => { ...$schemaOverrides[field], name: field, type: $schema[relField]?.columns?.[relSubField]?.type, + readonly: true, } }) } From 4aa9e08ec564083ca6ff0e38f0feb45767631fa3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:39:04 +0200 Subject: [PATCH 074/195] Don't show non visible even if previously configured --- .../frontend-core/src/components/grid/stores/datasource.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 399ec341ee..65e925fdf2 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,6 +83,10 @@ export const deriveStores = context => { const { field: relField, subField: relSubField } = $schemaOverrides[field].related + if (!$schema[relField]?.columns?.[relSubField]?.visible) { + return + } + enrichedSchema[field] = { ...$schemaOverrides[field], name: field, From e9db3d64e7a8914cb154c654417e74ddd642e799 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:50:11 +0200 Subject: [PATCH 075/195] Extra enrichement --- .../frontend-core/src/components/grid/stores/datasource.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index 65e925fdf2..a7a3083cde 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -83,7 +83,10 @@ export const deriveStores = context => { const { field: relField, subField: relSubField } = $schemaOverrides[field].related - if (!$schema[relField]?.columns?.[relSubField]?.visible) { + if ( + !$schema[relField].visible || + !$schema[relField]?.columns?.[relSubField]?.visible + ) { return } From 488165d85985056ad3b6034bb10aa87ec47347d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 14:00:53 +0200 Subject: [PATCH 076/195] Lint --- packages/frontend-core/src/components/grid/stores/datasource.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index a7a3083cde..bdd338618c 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -1,7 +1,6 @@ import { derived, get } from "svelte/store" import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch" import { memo } from "../../../utils" -import { FieldType } from "@budibase/types" export const createStores = () => { const definition = memo(null) From eed82075fd6c678a4b509a76a9e3e7ef1060603f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 09:52:09 +0200 Subject: [PATCH 077/195] Enrich visible columns --- .../src/components/grid/stores/columns.js | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 0073754a5d..9648b367f9 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -31,7 +31,7 @@ export const createStores = () => { } export const deriveStores = context => { - const { columns } = context + const { columns, enrichedSchema } = context // Derive a lookup map for all columns by name const columnLookupMap = derived(columns, $columns => { @@ -43,9 +43,38 @@ export const deriveStores = context => { }) // Derived list of columns which have not been explicitly hidden - const visibleColumns = derived(columns, $columns => { - return $columns.filter(col => col.visible) - }) + const visibleColumns = derived( + [columns, enrichedSchema], + ([$columns, $enrichedSchema]) => { + return $columns + .filter(col => col.visible) + .flatMap(c => { + const relatedColumns = [] + + const schemaColumns = $enrichedSchema?.[c.name]?.columns + if (schemaColumns) { + for (const relColumn of Object.keys(schemaColumns)) { + const relFieldSchema = schemaColumns[relColumn] + if (!relFieldSchema.visible) { + continue + } + relatedColumns.push({ + name: `${c.name}.${relColumn}`, + label: `${relColumn} (${c.name})`, + schema: relFieldSchema, + width: relFieldSchema.width || DefaultColumnWidth, + visible: relFieldSchema.visible ?? true, + readonly: relFieldSchema.readonly, + order: relFieldSchema.order, + conditions: relFieldSchema.conditions, + }) + } + } + + return [c, ...relatedColumns] + }) + } + ) // Split visible columns into their discrete types const displayColumn = derived(visibleColumns, $visibleColumns => { @@ -136,7 +165,7 @@ export const initialise = context => { .map(field => { const fieldSchema = $enrichedSchema[field] const oldColumn = $columns?.find(col => col.name === field) - let column = { + const column = { name: field, label: fieldSchema.displayName || field, schema: fieldSchema, From 4272b614e254927ddbea2bce2762ca4128854522 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 09:58:43 +0200 Subject: [PATCH 078/195] Cleanup --- .../GridColumnConfiguration/getColumns.js | 17 ----------------- .../client/src/components/app/GridBlock.svelte | 1 - 2 files changed, 18 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 7380e40de0..f5ac106162 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -8,7 +8,6 @@ const modernize = columns => { label: column.displayName, field: column.name, active: true, - related: column.related, })) } @@ -75,20 +74,6 @@ const getDefault = (schema = {}) => { }, ] - if (column.columns) { - for (const relColumn of Object.keys(column.columns).filter( - relColumn => column.columns[relColumn].visible !== false - )) { - columns.push({ - label: `${relColumn} (${column.name})`, - field: `${column.name}.${relColumn}`, - active: false, - order, - related: { field: column.name, subField: relColumn }, - }) - } - } - return columns }) @@ -104,7 +89,6 @@ const toGridFormat = draggableListColumns => { active: entry.active, width: entry.width, conditions: entry.conditions, - related: entry.related, })) } @@ -121,7 +105,6 @@ const toDraggableListFormat = (gridFormatColumns, createComponent) => { columnType: column.columnType, width: column.width, conditions: column.conditions, - related: column.related, }, {} ) diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte index 45aba66bab..30a35b0713 100644 --- a/packages/client/src/components/app/GridBlock.svelte +++ b/packages/client/src/components/app/GridBlock.svelte @@ -98,7 +98,6 @@ order: idx, conditions: column.conditions, visible: !!column.active, - related: column.related, } if (column.width) { overrides[column.field].width = column.width From 6d8505cee617cd3ebdc50d5edaf4d6733ba97688 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 10:18:43 +0200 Subject: [PATCH 079/195] Add tablecolumns --- .../grid/controls/ColumnsSettingButton.svelte | 12 ++-- .../src/components/grid/stores/columns.js | 69 ++++++++++--------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index 2a3ca139fc..75108870bd 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -4,13 +4,15 @@ import ColumnsSettingContent from "./ColumnsSettingContent.svelte" import { FieldPermissions } from "../../../constants" - const { columns, datasource } = getContext("grid") + const { tableColumns, datasource } = getContext("grid") let open = false let anchor - $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length - $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns" + $: anyRestricted = $tableColumns.filter( + col => !col.visible || col.readonly + ).length + $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns" $: permissions = $datasource.type === "viewV2" ? [ @@ -28,12 +30,12 @@ size="M" on:click={() => (open = !open)} selected={open || anyRestricted} - disabled={!$columns.length} + disabled={!$tableColumns.length} > {text} - + diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index 9648b367f9..b20f17fd4d 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -31,7 +31,7 @@ export const createStores = () => { } export const deriveStores = context => { - const { columns, enrichedSchema } = context + const { columns } = context // Derive a lookup map for all columns by name const columnLookupMap = derived(columns, $columns => { @@ -42,39 +42,15 @@ export const deriveStores = context => { return map }) + // Derived list of columns which are direct part of the table + const tableColumns = derived(columns, $columns => { + return $columns.filter(col => !col.related) + }) + // Derived list of columns which have not been explicitly hidden - const visibleColumns = derived( - [columns, enrichedSchema], - ([$columns, $enrichedSchema]) => { - return $columns - .filter(col => col.visible) - .flatMap(c => { - const relatedColumns = [] - - const schemaColumns = $enrichedSchema?.[c.name]?.columns - if (schemaColumns) { - for (const relColumn of Object.keys(schemaColumns)) { - const relFieldSchema = schemaColumns[relColumn] - if (!relFieldSchema.visible) { - continue - } - relatedColumns.push({ - name: `${c.name}.${relColumn}`, - label: `${relColumn} (${c.name})`, - schema: relFieldSchema, - width: relFieldSchema.width || DefaultColumnWidth, - visible: relFieldSchema.visible ?? true, - readonly: relFieldSchema.readonly, - order: relFieldSchema.order, - conditions: relFieldSchema.conditions, - }) - } - } - - return [c, ...relatedColumns] - }) - } - ) + const visibleColumns = derived(columns, $columns => { + return $columns.filter(col => col.visible) + }) // Split visible columns into their discrete types const displayColumn = derived(visibleColumns, $visibleColumns => { @@ -93,6 +69,7 @@ export const deriveStores = context => { }) return { + tableColumns, displayColumn, columnLookupMap, visibleColumns, @@ -183,6 +160,32 @@ export const initialise = context => { } return column }) + .flatMap(field => { + const relatedColumns = [] + + const schemaColumns = $enrichedSchema?.[field.name]?.columns + if (field.visible && schemaColumns) { + for (const relColumn of Object.keys(schemaColumns)) { + const relFieldSchema = schemaColumns[relColumn] + if (!relFieldSchema.visible) { + continue + } + relatedColumns.push({ + name: `${field.name}.${relColumn}`, + label: `${relColumn} (${field.name})`, + schema: relFieldSchema, + width: relFieldSchema.width || DefaultColumnWidth, + visible: relFieldSchema.visible ?? true, + readonly: relFieldSchema.readonly, + order: relFieldSchema.order, + conditions: relFieldSchema.conditions, + related: true, + }) + } + } + + return [field, ...relatedColumns] + }) .sort((a, b) => { // Display column should always come first if (a.name === primaryDisplay) { From 083e6ae15bb1a3d87b1cf48ff5a07dfd4241f918 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 12:31:00 +0200 Subject: [PATCH 080/195] Enrich at schema level --- .../src/components/grid/stores/columns.js | 27 +--------- .../src/components/grid/stores/datasource.js | 49 +++++++++---------- 2 files changed, 24 insertions(+), 52 deletions(-) diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js index b20f17fd4d..b4eba300ad 100644 --- a/packages/frontend-core/src/components/grid/stores/columns.js +++ b/packages/frontend-core/src/components/grid/stores/columns.js @@ -151,6 +151,7 @@ export const initialise = context => { readonly: fieldSchema.readonly, order: fieldSchema.order ?? oldColumn?.order, conditions: fieldSchema.conditions, + related: fieldSchema.related, } // Override a few properties for primary display if (field === primaryDisplay) { @@ -160,32 +161,6 @@ export const initialise = context => { } return column }) - .flatMap(field => { - const relatedColumns = [] - - const schemaColumns = $enrichedSchema?.[field.name]?.columns - if (field.visible && schemaColumns) { - for (const relColumn of Object.keys(schemaColumns)) { - const relFieldSchema = schemaColumns[relColumn] - if (!relFieldSchema.visible) { - continue - } - relatedColumns.push({ - name: `${field.name}.${relColumn}`, - label: `${relColumn} (${field.name})`, - schema: relFieldSchema, - width: relFieldSchema.width || DefaultColumnWidth, - visible: relFieldSchema.visible ?? true, - readonly: relFieldSchema.readonly, - order: relFieldSchema.order, - conditions: relFieldSchema.conditions, - related: true, - }) - } - } - - return [field, ...relatedColumns] - }) .sort((a, b) => { // Display column should always come first if (a.name === primaryDisplay) { diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js index bdd338618c..5ec94874dd 100644 --- a/packages/frontend-core/src/components/grid/stores/datasource.js +++ b/packages/frontend-core/src/components/grid/stores/datasource.js @@ -53,10 +53,31 @@ export const deriveStores = context => { if (!$schema) { return null } + + const schemaWithRelatedColumns = Object.keys($schema || {}).reduce( + (acc, c) => { + const field = $schema[c] + acc[c] = field + + if (field.columns) { + for (const relColumn of Object.keys(field.columns)) { + const name = `${field.name}.${relColumn}` + acc[name] = { + ...field.columns[relColumn], + name, + related: { field: c, subField: relColumn }, + } + } + } + return acc + }, + {} + ) + let enrichedSchema = {} - Object.keys($schema).forEach(field => { + Object.keys(schemaWithRelatedColumns).forEach(field => { enrichedSchema[field] = { - ...$schema[field], + ...schemaWithRelatedColumns[field], ...$schemaOverrides?.[field], ...$schemaMutations[field], } @@ -73,30 +94,6 @@ export const deriveStores = context => { } } }) - if ($schemaOverrides) { - Object.keys($schemaOverrides).forEach(field => { - if (!$schemaOverrides[field].related) { - return - } - - const { field: relField, subField: relSubField } = - $schemaOverrides[field].related - - if ( - !$schema[relField].visible || - !$schema[relField]?.columns?.[relSubField]?.visible - ) { - return - } - - enrichedSchema[field] = { - ...$schemaOverrides[field], - name: field, - type: $schema[relField]?.columns?.[relSubField]?.type, - readonly: true, - } - }) - } return enrichedSchema } ) From 4e83daf5d96fc734f74fe62150545a1130de729d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 12:41:59 +0200 Subject: [PATCH 081/195] Allow selecting columns in design --- .../GridColumnConfiguration.svelte | 4 ++- .../src/components/grid/stores/datasource.js | 24 +++-------------- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/schema.js | 27 +++++++++++++++++++ 4 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 packages/frontend-core/src/utils/schema.js diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte index 17cb171da5..b89c45046e 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte @@ -1,4 +1,5 @@ \ No newline at end of file diff --git a/packages/builder/src/pages/builder/portal/settings/index.svelte b/packages/builder/src/pages/builder/portal/settings/index.svelte index 09ead3e410..1448b43ec4 100644 --- a/packages/builder/src/pages/builder/portal/settings/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/index.svelte @@ -1,5 +1,10 @@ From ee962380b3d72c40b4f337ebd573e7ed22ab8f30 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 16:18:19 +0200 Subject: [PATCH 122/195] Handle undefined properly --- .../frontend-core/src/utils/relatedColumns.js | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js index 55f1bbbb00..9d389524fd 100644 --- a/packages/frontend-core/src/utils/relatedColumns.js +++ b/packages/frontend-core/src/utils/relatedColumns.js @@ -75,36 +75,32 @@ export function getRelatedTableValues(row, field, fromField) { fromField?.relationshipType === RelationshipType.ONE_TO_MANY let result = "" - try { - if (fromSingle) { - result = row[field.related.field]?.[0]?.[field.related.subField] - } else { - const parser = columnTypeManyParser[field.type] || (value => value) - result = parser( - row[field.related.field] - .flatMap(r => r[field.related.subField]) - .filter(i => i !== undefined && i !== null), - field - ) + if (fromSingle) { + result = row[field.related.field]?.[0]?.[field.related.subField] + } else { + const parser = columnTypeManyParser[field.type] || (value => value) - if ( - [ - FieldType.STRING, - FieldType.NUMBER, - FieldType.BIGINT, - FieldType.BOOLEAN, - FieldType.DATETIME, - FieldType.LONGFORM, - FieldType.BARCODEQR, - ].includes(field.type) - ) { - result = result.join(", ") - } + result = parser( + row[field.related.field] + ?.flatMap(r => r[field.related.subField]) + ?.filter(i => i !== undefined && i !== null), + field + ) + + if ( + [ + FieldType.STRING, + FieldType.NUMBER, + FieldType.BIGINT, + FieldType.BOOLEAN, + FieldType.DATETIME, + FieldType.LONGFORM, + FieldType.BARCODEQR, + ].includes(field.type) + ) { + result = result?.join(", ") } - } catch (e) { - result = "Not rendable" - console.error(e.message) } return result From c4c524c6ff35921f036b4ef9fdb051bf58ecc15f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:22:10 +0100 Subject: [PATCH 123/195] viewV2.spec.ts passsing in full --- packages/backend-core/src/sql/sql.ts | 5 +- .../api/controllers/row/ExternalRequest.ts | 22 +++- .../src/api/controllers/row/utils/basic.ts | 32 ++--- .../src/api/controllers/row/utils/sqlUtils.ts | 41 +++++-- .../src/api/controllers/row/utils/utils.ts | 5 +- .../src/api/routes/tests/viewV2.spec.ts | 114 +++++++++--------- .../src/sdk/app/rows/search/external.ts | 11 +- .../sdk/app/rows/search/internal/lucene.ts | 11 +- .../src/sdk/app/rows/search/internal/sqs.ts | 1 - .../src/utilities/rowProcessor/index.ts | 9 +- 10 files changed, 151 insertions(+), 100 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index bfb9d9bfbe..b7bf5bc102 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -859,7 +859,7 @@ class InternalBuilder { } addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { - let { sort } = this.query + let { sort, resource } = this.query const primaryKey = this.table.primary const tableName = getTableName(this.table) const aliases = this.query.tableAliases @@ -896,7 +896,8 @@ class InternalBuilder { // add sorting by the primary key if the result isn't already sorted by it, // to make sure result is deterministic - if (!sort || sort[primaryKey[0]] === undefined) { + const hasAggregations = (resource?.aggregations?.length ?? 0) > 0 + if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) { query = query.orderBy(`${aliased}.${primaryKey[0]}`) } return query diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index a52d7abcd1..4ce326c35f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,5 +1,6 @@ import dayjs from "dayjs" import { + Aggregation, AutoFieldSubType, AutoReason, Datasource, @@ -47,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" import { makeExternalQuery } from "../../../integrations/base/query" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" export interface ManyRelationship { tableId?: string @@ -682,12 +683,26 @@ export class ExternalRequest { } } } + if ( operation === Operation.DELETE && (filters == null || Object.keys(filters).length === 0) ) { throw "Deletion must be filtered" } + + let aggregations: Aggregation[] = [] + if (sdk.views.isView(this.source)) { + const calculationFields = helpers.views.calculationFields(this.source) + for (const [key, field] of Object.entries(calculationFields)) { + aggregations.push({ + name: key, + field: field.field, + calculationType: field.calculationType, + }) + } + } + let json: QueryJson = { endpoint: { datasourceId: this.datasource._id!, @@ -697,10 +712,11 @@ export class ExternalRequest { resource: { // have to specify the fields to avoid column overlap (for SQL) fields: isSql - ? buildSqlFieldList(table, this.tables, { + ? await buildSqlFieldList(this.source, this.tables, { relationships: incRelationships, }) : [], + aggregations, }, filters, sort, @@ -748,7 +764,7 @@ export class ExternalRequest { } const output = await sqlOutputProcessing( response, - table, + this.source, this.tables, relationships ) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9c86a165ab..23565670ba 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -100,8 +100,10 @@ export async function basicProcessing({ sqs?: boolean }): Promise { let table: Table + let isCalculationView = false if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) + isCalculationView = helpers.views.isCalculationView(source) } else { table = source } @@ -132,20 +134,22 @@ export async function basicProcessing({ } let columns: string[] = Object.keys(table.schema) - if (!sqs) { - thisRow._id = generateIdForRow(row, table, isLinked) - thisRow.tableId = table._id - thisRow._rev = "rev" - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - } else { - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - thisRow[internalColumn] = extractFieldValue({ - row, - tableName: table._id!, - fieldName: internalColumn, - isLinked, - }) + if (!isCalculationView) { + if (!sqs) { + thisRow._id = generateIdForRow(row, table, isLinked) + thisRow.tableId = table._id + thisRow._rev = "rev" + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + } else { + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { + thisRow[internalColumn] = extractFieldValue({ + row, + tableName: table._id!, + fieldName: internalColumn, + isLinked, + }) + } } } for (let col of columns) { diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index d51972b2c9..36521b10c6 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -9,9 +9,12 @@ import { RelationshipsJson, Row, Table, + ViewV2, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" import { generateJunctionTableID } from "../../../../db/utils" +import sdk from "../../../../sdk" +import { helpers } from "@budibase/shared-core" type TableMap = Record @@ -109,11 +112,12 @@ export function buildInternalRelationships( * Creating the specific list of fields that we desire, and excluding the ones that are no use to us * is more performant and has the added benefit of protecting against this scenario. */ -export function buildSqlFieldList( - table: Table, +export async function buildSqlFieldList( + source: Table | ViewV2, tables: TableMap, opts?: { relationships: boolean } ) { + const { relationships } = opts || {} function extractRealFields(table: Table, existing: string[] = []) { return Object.entries(table.schema) .filter( @@ -124,22 +128,33 @@ export function buildSqlFieldList( ) .map(column => `${table.name}.${column[0]}`) } - let fields = extractRealFields(table) + + let fields: string[] = [] + if (sdk.views.isView(source)) { + fields = Object.keys(helpers.views.basicFields(source)).filter( + key => source.schema?.[key]?.visible !== false + ) + } else { + fields = extractRealFields(source) + } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + for (let field of Object.values(table.schema)) { - if ( - field.type !== FieldType.LINK || - !opts?.relationships || - !field.tableId - ) { + if (field.type !== FieldType.LINK || !relationships || !field.tableId) { continue } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - const linkTable = tables[linkTableName] - if (linkTable) { - const linkedFields = extractRealFields(linkTable, fields) - fields = fields.concat(linkedFields) + const { tableName } = breakExternalTableId(field.tableId) + if (tables[tableName]) { + fields = fields.concat(extractRealFields(tables[tableName], fields)) } } + return fields } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index b673106d26..4188fcced3 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -19,6 +19,7 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" +import { helpers } from "@budibase/shared-core" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -121,8 +122,10 @@ export async function sqlOutputProcessing( } let table: Table + let isCalculationView = false if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) + isCalculationView = helpers.views.isCalculationView(source) } else { table = source } @@ -131,7 +134,7 @@ export async function sqlOutputProcessing( for (let row of rows) { if (opts?.sqs) { row._id = getInternalRowId(row, table) - } else if (row._id == null) { + } else if (row._id == null && !isCalculationView) { row._id = generateIdForRow(row, table) } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index ba2e93afbf..883b25aca1 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,16 +37,15 @@ import { setEnv as setCoreEnv, env, } from "@budibase/backend-core" -import sdk from "../../../sdk" describe.each([ - // ["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2362,63 +2361,70 @@ describe.each([ }) }) - describe("calculations", () => { - let table: Table - let rows: Row[] + !isLucene && + describe("calculations", () => { + let table: Table + let rows: Row[] - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, }, - price: { - type: FieldType.NUMBER, - name: "price", + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", }, }, }) - ) - rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, { - quantity: generator.natural({ min: 1, max: 10 }), - price: generator.natural({ min: 1, max: 10 }), - }) + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) ) - ) - }) - it("should be able to search by calculations", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - "Quantity Sum": { - visible: true, - calculationType: CalculationType.SUM, - field: "quantity", - }, - }, + // Calculation views do not return rows that can be linked back to + // the source table, and so should not have an _id field. + for (const row of response.rows) { + expect("_id" in row).toBe(false) + } }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), - }), - ]) - ) }) - }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 925b472a48..fa8961461b 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -127,10 +127,13 @@ export async function search( } } - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS] - processed = processed.map((r: any) => pick(r, fields)) - } + const visibleFields = + options.fields || + Object.keys(source.schema || {}).filter( + key => source.schema?.[key].visible !== false + ) + const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS] + processed = processed.map((r: any) => pick(r, allowedFields)) // need wrapper object for bookmarks etc when paginating const response: SearchResponse = { rows: processed, hasNextPage } diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 694c660625..953fb90c1f 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -62,10 +62,13 @@ export async function search( response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] - response.rows = response.rows.map((r: any) => pick(r, fields)) - } + const visibleFields = + options.fields || + Object.keys(source.schema || {}).filter( + key => source.schema?.[key].visible !== false + ) + const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS] + response.rows = response.rows.map((r: any) => pick(r, allowedFields)) response.rows = await outputProcessing(source, response.rows, { squash: true, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 26e8f59303..c185ef18dd 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -460,7 +460,6 @@ export async function search( let finalRows = await outputProcessing(source, processed, { preserveLinks: true, squash: true, - aggregations, }) const visibleFields = diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 24c8d11bd1..818c98a84e 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,7 +11,6 @@ import { import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { - Aggregation, AutoFieldSubType, FieldType, IdentityType, @@ -263,7 +262,6 @@ export async function outputProcessing( preserveLinks?: boolean fromRow?: Row skipBBReferences?: boolean - aggregations?: Aggregation[] } = { squash: true, preserveLinks: false, @@ -411,8 +409,11 @@ export async function outputProcessing( f.toLowerCase() ) - for (const aggregation of opts.aggregations || []) { - fields.push(aggregation.name.toLowerCase()) + if (sdk.views.isView(source)) { + const aggregations = helpers.views.calculationFields(source) + for (const key of Object.keys(aggregations)) { + fields.push(key.toLowerCase()) + } } for (const row of enriched) { From 7c6c03c80b2ad15944b6c830f1abc6fd213d7c8e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:32:21 +0100 Subject: [PATCH 124/195] Rename ViewUIFieldMetadata -> ViewFieldMetadata to match master. --- packages/server/src/api/controllers/view/viewsV2.ts | 12 ++++++------ packages/server/src/api/routes/tests/viewV2.spec.ts | 4 ++-- packages/server/src/db/linkedRows/index.ts | 4 ++-- packages/shared-core/src/helpers/views.ts | 10 +++++----- packages/types/src/documents/app/view.ts | 10 +++++----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index fddcef97a2..7f6f638541 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -7,17 +7,17 @@ import { ViewResponse, ViewResponseEnriched, ViewV2, - BasicViewUIFieldMetadata, + BasicViewFieldMetadata, ViewCalculationFieldMetadata, RelationSchemaField, - ViewUIFieldMetadata, + ViewFieldMetadata, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" function stripUnknownFields( - field: BasicViewUIFieldMetadata -): RequiredKeys { + field: BasicViewFieldMetadata +): RequiredKeys { if (helpers.views.isCalculationField(field)) { const strippedField: RequiredKeys = { order: field.order, @@ -31,7 +31,7 @@ function stripUnknownFields( } return strippedField } else { - const strippedField: RequiredKeys = { + const strippedField: RequiredKeys = { order: field.order, width: field.width, visible: field.visible, @@ -83,7 +83,7 @@ async function parseSchema(view: CreateViewRequest) { p[fieldName] = fieldSchema return p - }, {} as Record>) + }, {} as Record>) return finalViewSchema } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 883b25aca1..3712efa5a4 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,7 +22,7 @@ import { RelationshipType, TableSchema, RenameColumn, - ViewUIFieldMetadata, + ViewFieldMetadata, FeatureFlag, BBReferenceFieldSubType, } from "@budibase/types" @@ -1154,7 +1154,7 @@ describe.each([ const createView = async ( tableId: string, - schema: Record + schema: Record ) => await config.api.viewV2.create({ name: generator.guid(), diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index ed660288c8..41cbc5b9c1 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -20,7 +20,7 @@ import { Row, Table, TableSchema, - ViewUIFieldMetadata, + ViewFieldMetadata, ViewV2, } from "@budibase/types" import sdk from "../../sdk" @@ -263,7 +263,7 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} + let viewSchema: Record = {} if (sdk.views.isView(source)) { if (helpers.views.isCalculationView(source)) { return enriched diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts index c65bc4882d..f41c66adc8 100644 --- a/packages/shared-core/src/helpers/views.ts +++ b/packages/shared-core/src/helpers/views.ts @@ -1,20 +1,20 @@ import { - BasicViewUIFieldMetadata, + BasicViewFieldMetadata, ViewCalculationFieldMetadata, - ViewUIFieldMetadata, + ViewFieldMetadata, ViewV2, } from "@budibase/types" import { pickBy } from "lodash" export function isCalculationField( - field: ViewUIFieldMetadata + field: ViewFieldMetadata ): field is ViewCalculationFieldMetadata { return "calculationType" in field } export function isBasicViewField( - field: ViewUIFieldMetadata -): field is BasicViewUIFieldMetadata { + field: ViewFieldMetadata +): field is BasicViewFieldMetadata { return !isCalculationField(field) } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 74b8f61f59..a957564039 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -33,7 +33,7 @@ export interface View { groupBy?: string } -export interface BasicViewUIFieldMetadata extends UIFieldMetadata { +export interface BasicViewFieldMetadata extends UIFieldMetadata { readonly?: boolean columns?: Record } @@ -42,13 +42,13 @@ export interface RelationSchemaField extends UIFieldMetadata { readonly?: boolean } -export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata { +export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata { calculationType: CalculationType field: string } -export type ViewUIFieldMetadata = - | BasicViewUIFieldMetadata +export type ViewFieldMetadata = + | BasicViewFieldMetadata | ViewCalculationFieldMetadata export enum CalculationType { @@ -71,7 +71,7 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: Record } export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema From 25a2e02a90acf6b07b71dce4cde62d4899ce7cdd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:40:25 +0100 Subject: [PATCH 125/195] Remove needless table copy. --- packages/server/src/api/controllers/row/internal.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f5cb42f81d..bb9de6ce52 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -53,11 +53,8 @@ export async function patch(ctx: UserCtx) { combinedRow[key] = inputs[key] } - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(table) - // this returns the table and row incase they have been updated - let row = await inputProcessing(ctx.user?._id, tableClone, combinedRow) + let row = await inputProcessing(ctx.user?._id, source, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, source, From 26a27ff70f1a0e9a363ce9e9d19cae123f1e7776 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:48:44 +0100 Subject: [PATCH 126/195] Remove needless table copy. --- packages/server/src/sdk/app/rows/internal.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index 13054f43f4..997af3c907 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -34,10 +34,7 @@ export async function save( inputs._id = db.generateRowID(inputs.tableId) } - // need to copy the table so it can be differenced on way out - const sourceClone = cloneDeep(source) - - let row = await inputProcessing(userId, sourceClone, inputs) + let row = await inputProcessing(userId, source, inputs) const validateResult = await sdk.rows.utils.validate({ row, From ec6fa5f79b2dda85dbb7043291fecf6bc59ec3f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:50:49 +0100 Subject: [PATCH 127/195] Return SQS error to prevoius state. --- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index c185ef18dd..f0e47ce2b7 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -495,6 +495,6 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - throw err + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } From ae8a8645660518404dba217013627c126c2783a5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:51:21 +0100 Subject: [PATCH 128/195] Collapse duplicated key names. --- packages/server/src/sdk/app/rows/search/tests/utils.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index daa658455b..33594a621c 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - tableId: tableId, + tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - tableId: tableId, + tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - tableId: tableId, + tableId, query: { equal: { "1:user": email, From 743140eeaee5ea0d6b2269470e98d63411174d7f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:55:37 +0100 Subject: [PATCH 129/195] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index afe53b4be5..6c82625fdc 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit afe53b4be55ac6f93dc9486596835412b2673f90 +Subproject commit 6c82625fdca3318e9fdbb61b8c2e9e78e5f8ace9 From d7715e4f5c84ec0ccdde40a20cc93836c96c3660 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 17:00:44 +0200 Subject: [PATCH 130/195] Don't allow related columns as display or sorting --- .../DataTable/modals/CreateEditColumn.svelte | 5 ++--- .../backend/TableNavigator/TableDataImport.svelte | 7 ++++--- .../src/components/grid/cells/HeaderCell.svelte | 6 +++--- packages/frontend-core/src/utils/index.js | 1 + packages/frontend-core/src/utils/table.js | 14 ++++++++++++++ 5 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 packages/frontend-core/src/utils/table.js diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a956d09ee6..0130c39715 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -19,7 +19,6 @@ helpers, PROTECTED_INTERNAL_COLUMNS, PROTECTED_EXTERNAL_COLUMNS, - canBeDisplayColumn, canHaveDefaultColumn, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" @@ -43,7 +42,7 @@ SourceName, } from "@budibase/types" import RelationshipSelector from "components/common/RelationshipSelector.svelte" - import { RowUtils } from "@budibase/frontend-core" + import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" @@ -166,7 +165,7 @@ : availableAutoColumns // used to select what different options can be displayed for column type $: canBeDisplay = - canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn + canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn $: canHaveDefault = isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) $: canBeRequired = diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte index 4c9f4dd10f..5804dc3172 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte @@ -1,7 +1,8 @@ \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + From 133ce9e784a3187fde6a80ce5098674c3b2fe54d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 26 Sep 2024 17:21:18 +0100 Subject: [PATCH 136/195] update colour on tags --- .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index a95d14b273..c63f894d56 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -112,7 +112,7 @@ .tag { display: flex; - color: var(--spectrum-body-m-text-color); + color: #FFFFFF; padding: 4px 8px; justify-content: center; align-items: center; From 264b10f3f398a5e66189f5503f8802b0cca0f141 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 17:22:11 +0100 Subject: [PATCH 137/195] Fix isView. --- packages/server/src/sdk/app/views/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4aea36fa2a..cd0fdf077e 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -53,7 +53,7 @@ export async function getTable(view: string | ViewV2): Promise
{ } export function isView(view: any): view is ViewV2 { - return view._id && docIds.isViewId(view._id) && view.version === 2 + return view.id && docIds.isViewId(view.id) && view.version === 2 } async function guardCalculationViewSchema( From 98a2da20b4be8623c524aac3221677c45f7b3aaf Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 26 Sep 2024 17:25:13 +0100 Subject: [PATCH 138/195] lint --- .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte | 2 +- .../builder/src/pages/builder/portal/settings/ai/index.svelte | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index c63f894d56..fb040204c2 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -112,7 +112,7 @@ .tag { display: flex; - color: #FFFFFF; + color: #ffffff; padding: 4px 8px; justify-content: center; align-items: center; diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 54b1bd60da..42de9f19ae 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -27,7 +27,6 @@ let editingUuid $: isCloud = $admin.cloud - $: budibaseAIEnabled = $licensing.budibaseAIEnabled $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled async function fetchAIConfig() { From aa044b94cf4615f778d27b1cd6f422ff65bca68b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 26 Sep 2024 21:09:47 +0200 Subject: [PATCH 139/195] Don't allow sort columns as display or sorting --- .../src/components/backend/DataTable/Table.svelte | 4 ++-- .../settings/controls/SortableFieldSelect.svelte | 4 ++-- .../components/app/deprecated/table/Table.svelte | 4 ++-- .../src/components/grid/cells/HeaderCell.svelte | 7 +++---- .../components/grid/controls/SortButton.svelte | 4 ++-- packages/frontend-core/src/utils/table.js | 15 ++++++++++++++- 6 files changed, 25 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index f7eccd5242..e09dd4bd39 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -6,7 +6,7 @@ import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import RoleCell from "./cells/RoleCell.svelte" import { createEventDispatcher } from "svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" export let schema = {} export let data = [] @@ -31,7 +31,7 @@ acc[key] = typeof schema[key] === "string" ? { type: schema[key] } : schema[key] - if (!canBeSortColumn(acc[key].type)) { + if (!canBeSortColumn(acc[key])) { acc[key].sortable = false } return acc diff --git a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte index d51493616e..0f9290ec8d 100644 --- a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte @@ -3,7 +3,7 @@ import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" import { selectedScreen } from "stores/builder" import { createEventDispatcher } from "svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" export let componentInstance = {} export let value = "" @@ -17,7 +17,7 @@ const getSortableFields = schema => { return Object.entries(schema || {}) - .filter(entry => canBeSortColumn(entry[1].type)) + .filter(entry => canBeSortColumn(entry[1])) .map(entry => entry[0]) } diff --git a/packages/client/src/components/app/deprecated/table/Table.svelte b/packages/client/src/components/app/deprecated/table/Table.svelte index ac3d88d29c..ca97a012bc 100644 --- a/packages/client/src/components/app/deprecated/table/Table.svelte +++ b/packages/client/src/components/app/deprecated/table/Table.svelte @@ -2,7 +2,7 @@ import { getContext, onDestroy } from "svelte" import { Table } from "@budibase/bbui" import SlotRenderer from "./SlotRenderer.svelte" - import { canBeSortColumn } from "@budibase/shared-core" + import { canBeSortColumn } from "@budibase/frontend-core" import Provider from "components/context/Provider.svelte" export let dataProvider @@ -146,7 +146,7 @@ return } newSchema[columnName] = schema[columnName] - if (!canBeSortColumn(schema[columnName].type)) { + if (!canBeSortColumn(schema[columnName])) { newSchema[columnName].sortable = false } diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte index 23e19af96b..00b2cbabf3 100644 --- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte @@ -1,7 +1,6 @@ -
+ + + {#if aiCronPrompt} +
+ +
+ {/if} +
+ {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} {/if} -
- - {#if presets} - =0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20031,33 +20045,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21620,7 +21614,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21630,14 +21624,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22166,6 +22152,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22940,33 +22934,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22976,11 +22947,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From f28cb1badbb503e548d4faf871bb789bba9f0d05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 15:18:15 +0100 Subject: [PATCH 176/195] Another slight change. --- packages/server/src/sdk/app/rows/search/utils.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 8d96c2f8f9..1dba420a28 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -105,11 +105,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } } } - return filters + return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters( - dataFilters.recurseLogicalOperators(options.query, checkFilters) - ) + options.query = checkFilters(options.query) return options } From 28bb3215115aada6a10c50202eae175916c1343d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Sep 2024 15:36:49 +0100 Subject: [PATCH 177/195] Set view permissions to explicit roles from the parent table --- .../server/src/api/controllers/permission.ts | 108 ++---------------- .../src/api/routes/tests/permissions.spec.ts | 15 +++ .../src/api/routes/tests/rowAction.spec.ts | 9 ++ .../src/api/routes/tests/viewV2.spec.ts | 5 + .../server/src/sdk/app/permissions/index.ts | 98 +++++++++++++++- packages/server/src/sdk/app/views/index.ts | 29 ++++- 6 files changed, 162 insertions(+), 102 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 66a3254348..c7afb6a351 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,9 +1,7 @@ import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, - Database, Role, - PermissionLevel, GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, @@ -12,107 +10,15 @@ import { RemovePermissionRequest, RemovePermissionResponse, } from "@budibase/types" -import { getRoleParams } from "../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../utilities/security" -import { removeFromArray } from "../../utilities" import sdk from "../../sdk" - -const enum PermissionUpdateType { - REMOVE = "remove", - ADD = "add", -} +import { PermissionUpdateType } from "../../sdk/app/permissions" const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// utility function to stop this repetition - permissions always stored under roles -async function getAllDBRoles(db: Database) { - const body = await db.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return body.rows.map(row => row.doc!) -} - -async function updatePermissionOnRole( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - updateType: PermissionUpdateType -) { - const db = context.getAppDB() - const remove = updateType === PermissionUpdateType.REMOVE - const isABuiltin = roles.isBuiltin(roleId) - const dbRoleId = roles.getDBRoleID(roleId) - const dbRoles = await getAllDBRoles(db) - const docUpdates: Role[] = [] - - // the permission is for a built in, make sure it exists - if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { - const builtin = roles.getBuiltinRoles()[roleId] - builtin._id = roles.getDBRoleID(builtin._id!) - dbRoles.push(builtin) - } - - // now try to find any roles which need updated, e.g. removing the - // resource from another role and then adding to the new role - for (let role of dbRoles) { - let updated = false - const rolePermissions: Record = role.permissions - ? role.permissions - : {} - // make sure its an array, also handle migrating - if ( - !rolePermissions[resourceId] || - !Array.isArray(rolePermissions[resourceId]) - ) { - rolePermissions[resourceId] = - typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId] as unknown as PermissionLevel] - : [] - } - // handle the removal/updating the role which has this permission first - // the updating (role._id !== dbRoleId) is required because a resource/level can - // only be permitted in a single role (this reduces hierarchy confusion and simplifies - // the general UI for this, rather than needing to show everywhere it is used) - if ( - (role._id !== dbRoleId || remove) && - rolePermissions[resourceId].indexOf(level) !== -1 - ) { - removeFromArray(rolePermissions[resourceId], level) - updated = true - } - // handle the adding, we're on the correct role, at it to this - if (!remove && role._id === dbRoleId) { - const set = new Set(rolePermissions[resourceId]) - rolePermissions[resourceId] = [...set.add(level)] - updated = true - } - // handle the update, add it to bulk docs to perform at end - if (updated) { - role.permissions = rolePermissions - docUpdates.push(role) - } - } - - const response = await db.bulkDocs(docUpdates) - return response.map(resp => { - const version = docUpdates.find(role => role._id === resp.id)?.version - const _id = roles.getExternalRoleID(resp.id, version) - return { - _id, - rev: resp.rev, - error: resp.error, - reason: resp.reason, - } - }) -} - export function fetchBuiltin(ctx: UserCtx) { ctx.body = Object.values(permissions.getBuiltinPermissions()) } @@ -124,7 +30,7 @@ export function fetchLevels(ctx: UserCtx) { export async function fetch(ctx: UserCtx) { const db = context.getAppDB() - const dbRoles: Role[] = await getAllDBRoles(db) + const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db) let permissions: any = {} // create an object with structure role ID -> resource ID -> level for (let role of dbRoles) { @@ -186,12 +92,18 @@ export async function getDependantResources( export async function addPermission(ctx: UserCtx) { const params: AddPermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.ADD + ) } export async function removePermission( ctx: UserCtx ) { const params: RemovePermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.REMOVE + ) } diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 0f059998ae..b148d6fde1 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -125,6 +125,13 @@ describe("/permission", () => { }) it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => { + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: view.id, + level: PermissionLevel.READ, + }) + // replicate changes before checking permissions await config.publish() @@ -138,6 +145,14 @@ describe("/permission", () => { resourceId: table._id, level: PermissionLevel.READ, }) + + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: view.id, + level: PermissionLevel.READ, + }) + // replicate changes before checking permissions await config.publish() diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index ef7d2afbba..4fe248984a 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -826,11 +826,20 @@ describe("/rowsActions", () => { ) ).id + // Allow row action on view await config.api.rowAction.setViewPermission( tableId, viewId, rowAction.id ) + + // Delete explicit view permissions so they inherit table permissions + await config.api.permission.revoke({ + roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission + level: PermissionLevel.READ, + resourceId: viewId, + }) + return { permissionResource: tableId, triggerResouce: viewId } }, ], diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 3712efa5a4..aab846e704 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2460,6 +2460,11 @@ describe.each([ level: PermissionLevel.READ, resourceId: table._id!, }) + await config.api.permission.revoke({ + roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission + level: PermissionLevel.READ, + resourceId: view.id, + }) await config.publish() const response = await config.api.viewV2.publicSearch(view.id) diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index d5e4aefe3a..2c3c0af95b 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,22 +1,30 @@ -import { db, docIds, roles } from "@budibase/backend-core" +import { db, roles, context, docIds } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, + Role, + Database, } from "@budibase/types" -import { extractViewInfoFromID } from "../../../db/utils" +import { extractViewInfoFromID, getRoleParams } from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../../utilities/security" import sdk from "../../../sdk" import { isV2 } from "../views" +import { removeFromArray } from "../../../utilities" type ResourcePermissions = Record< string, { role: string; type: PermissionSource } > +export const enum PermissionUpdateType { + REMOVE = "remove", + ADD = "add", +} + export async function getInheritablePermissions( resourceId: string ): Promise { @@ -100,3 +108,89 @@ export async function getDependantResources( return } + +export async function updatePermissionOnRole( + { + roleId, + resourceId, + level, + }: { roleId: string; resourceId: string; level: PermissionLevel }, + updateType: PermissionUpdateType +) { + const db = context.getAppDB() + const remove = updateType === PermissionUpdateType.REMOVE + const isABuiltin = roles.isBuiltin(roleId) + const dbRoleId = roles.getDBRoleID(roleId) + const dbRoles = await getAllDBRoles(db) + const docUpdates: Role[] = [] + + // the permission is for a built in, make sure it exists + if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { + const builtin = roles.getBuiltinRoles()[roleId] + builtin._id = roles.getDBRoleID(builtin._id!) + dbRoles.push(builtin) + } + + // now try to find any roles which need updated, e.g. removing the + // resource from another role and then adding to the new role + for (let role of dbRoles) { + let updated = false + const rolePermissions: Record = role.permissions + ? role.permissions + : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId] as unknown as PermissionLevel] + : [] + } + // handle the removal/updating the role which has this permission first + // the updating (role._id !== dbRoleId) is required because a resource/level can + // only be permitted in a single role (this reduces hierarchy confusion and simplifies + // the general UI for this, rather than needing to show everywhere it is used) + if ( + (role._id !== dbRoleId || remove) && + rolePermissions[resourceId].indexOf(level) !== -1 + ) { + removeFromArray(rolePermissions[resourceId], level) + updated = true + } + // handle the adding, we're on the correct role, at it to this + if (!remove && role._id === dbRoleId) { + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] + updated = true + } + // handle the update, add it to bulk docs to perform at end + if (updated) { + role.permissions = rolePermissions + docUpdates.push(role) + } + } + + const response = await db.bulkDocs(docUpdates) + return response.map(resp => { + const version = docUpdates.find(role => role._id === resp.id)?.version + const _id = roles.getExternalRoleID(resp.id, version) + return { + _id, + rev: resp.rev, + error: resp.error, + reason: resp.reason, + } + }) +} + +// utility function to stop this repetition - permissions always stored under roles +export async function getAllDBRoles(db: Database) { + const body = await db.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + return body.rows.map(row => row.doc!) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index cd0fdf077e..24e4da3172 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,5 +1,6 @@ import { FieldType, + PermissionLevel, RelationSchemaField, RenameColumn, Table, @@ -9,7 +10,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { context, docIds, HTTPError } from "@budibase/backend-core" +import { context, docIds, HTTPError, roles } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -22,6 +23,7 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" +import { PermissionUpdateType, updatePermissionOnRole } from "../permissions" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -191,7 +193,30 @@ export async function create( ): Promise { await guardViewSchema(tableId, viewRequest) - return pickApi(tableId).create(tableId, viewRequest) + const view = await pickApi(tableId).create(tableId, viewRequest) + + // Set permissions to be the same as the table + const tablePerms = await sdk.permissions.getResourcePerms(tableId) + const readRole = tablePerms[PermissionLevel.READ]?.role + const writeRole = tablePerms[PermissionLevel.WRITE]?.role + await updatePermissionOnRole( + { + roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.READ, + }, + PermissionUpdateType.ADD + ) + await updatePermissionOnRole( + { + roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.WRITE, + }, + PermissionUpdateType.ADD + ) + + return view } export async function update(tableId: string, view: ViewV2): Promise { From 01415fb06673e3834d464d35226820ea22789352 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 15:43:58 +0100 Subject: [PATCH 178/195] Quick fix. --- packages/shared-core/src/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 45e9a7c6d0..ef0500b01a 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -124,7 +124,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters[logical]) { + if (filters?.[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) From f6649b294b1b62e54009a280d1ccaa0bdcb6b6ce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Sep 2024 15:54:01 +0100 Subject: [PATCH 179/195] Remove an implicit any from removeInvalidFilters. --- packages/server/src/sdk/app/rows/queryUtils.ts | 13 +++++++++---- packages/types/src/sdk/search.ts | 10 ++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 7ef776a989..c4f4a1eb2c 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -16,11 +16,11 @@ export const removeInvalidFilters = ( validFields = validFields.map(f => f.toLowerCase()) for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { - const filter = result[filterKey] - if (!filter || typeof filter !== "object") { - continue - } if (isLogicalSearchOperator(filterKey)) { + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } const resultingConditions: SearchFilters[] = [] for (const condition of filter.conditions) { const resultingCondition = removeInvalidFilters(condition, validFields) @@ -36,6 +36,11 @@ export const removeInvalidFilters = ( continue } + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } + for (const columnKey of Object.keys(filter)) { const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map( c => c.toLowerCase() diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 1d5b36031c..647a9e7d00 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -68,6 +68,8 @@ type RangeFilter = Record< [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } +type LogicalFilter = { conditions: SearchFilters[] } + export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter export interface SearchFilters { @@ -92,12 +94,8 @@ export interface SearchFilters { // specific document type (such as just rows) documentType?: DocumentType - [LogicalOperator.AND]?: { - conditions: SearchFilters[] - } - [LogicalOperator.OR]?: { - conditions: SearchFilters[] - } + [LogicalOperator.AND]?: LogicalFilter + [LogicalOperator.OR]?: LogicalFilter } export type SearchFilterKey = keyof Omit< From 8462b4e839dde7f138f97a5361ca9eb45510eda1 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 30 Sep 2024 16:10:39 +0000 Subject: [PATCH 180/195] Bump version to 2.32.9 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 10d36c9eaf..5a279b3e44 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.8", + "version": "2.32.9", "npmClient": "yarn", "packages": [ "packages/*", From f31c7c3487d2b2cd91454f1e89a9be12d2ddae7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 10:55:25 +0200 Subject: [PATCH 181/195] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aab846e704..09273abdce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1738,6 +1738,40 @@ describe.each([ }) }) + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + it("views without data can be returned", async () => { const response = await config.api.viewV2.search(view.id) expect(response.rows).toHaveLength(0) From 4b65ce4f8b51e55c9dc33de08337c8b8e0cbd7dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 1 Oct 2024 09:31:57 +0000 Subject: [PATCH 182/195] Bump version to 2.32.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5a279b3e44..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.9", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", From 987a24fabc85d76bb3332e364d877a4349083692 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 11:48:14 +0100 Subject: [PATCH 183/195] wip --- packages/backend-core/src/sql/sql.ts | 87 ++++++++++++------- .../src/api/routes/tests/viewV2.spec.ts | 39 +++++++-- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b7bf5bc102..54ca5a0135 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -71,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) { }) } -function getTableName(table?: Table): string | undefined { - // SQS uses the table ID rather than the table name - if ( - table?.sourceType === TableSourceType.INTERNAL || - table?.sourceId === INTERNAL_TABLE_SOURCE_ID - ) { - return table?._id - } else { - return table?.name - } -} - function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { if (Array.isArray(query)) { return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery) @@ -180,15 +168,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] @@ -813,17 +799,39 @@ class InternalBuilder { return query } + getTableName(t?: Table | string): string { + let table: Table + if (typeof t === "string") { + if (!this.query.meta.tables?.[t]) { + throw new Error(`Table ${t} not found`) + } + table = this.query.meta.tables[t] + } else if (t) { + table = t + } else { + table = this.table + } + + let name = table.name + if ( + (table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID) && + table._id + ) { + // SQS uses the table ID rather than the table name + name = table._id + } + const aliases = this.query.tableAliases || {} + return aliases[name] ? aliases[name] : name + } + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { - const primary = this.table.primary - const aliases = this.query.tableAliases - const aliased = - this.table.name && aliases?.[this.table.name] - ? aliases[this.table.name] - : this.table.name - if (!primary) { + if (!this.table.primary) { throw new Error("SQL counting requires primary key to be supplied") } - return query.countDistinct(`${aliased}.${primary[0]} as total`) + return query.countDistinct( + `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + ) } addAggregations( @@ -831,8 +839,9 @@ class InternalBuilder { aggregations: Aggregation[] ): Knex.QueryBuilder { const fields = this.query.resource?.fields || [] + const tableName = this.getTableName() if (fields.length > 0) { - query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + query = query.groupBy(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType @@ -861,10 +870,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort, resource } = this.query const primaryKey = this.table.primary - const tableName = getTableName(this.table) - const aliases = this.query.tableAliases - const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name + const aliased = this.getTableName() if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -1508,18 +1514,35 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } + private getTableName( + table: Table, + aliases?: Record + ): string | undefined { + let name = table.name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + if (!table._id) { + return + } + // SQS uses the table ID rather than the table name + name = table._id + } + return aliases?.[name] ? aliases[name] : name + } + convertJsonStringColumns>( table: Table, results: T[], aliases?: Record ): T[] { - const tableName = getTableName(table) + const tableName = this.getTableName(table, aliases) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = (tableName && aliases?.[tableName]) || tableName - const fullName = `${aliasedTableName}.${name}` + const fullName = `${tableName}.${name}` for (let row of results) { if (typeof row[fullName as keyof T] === "string") { row[fullName as keyof T] = JSON.parse(row[fullName]) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 09273abdce..0f4e6c961c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - ["lucene", undefined], - ["sqs", undefined], + // ["lucene", undefined], + // ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2458,6 +2458,33 @@ describe.each([ expect("_id" in row).toBe(false) } }) + + it.only("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + quantity: { + visible: true, + field: "quantity", + }, + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + for (const row of response.rows) { + expect(row["quantity"]).toBeGreaterThan(0) + expect(row["Total Price"]).toBeGreaterThan(0) + } + }) }) }) From 84f7a477a1a39953ce11bc3aeedce5ec0e872a34 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 12:00:03 +0100 Subject: [PATCH 184/195] Fix the binding drawer for default values. --- .../DataTable/modals/CreateEditColumn.svelte | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..a1bd54715b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -21,6 +21,7 @@ PROTECTED_EXTERNAL_COLUMNS, canHaveDefaultColumn, } from "@budibase/shared-core" + import { makePropSafe } from "@budibase/string-templates" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -46,6 +47,7 @@ import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" + import { getUserBindings } from "dataBinding" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -191,6 +193,19 @@ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: defaultValueBindings = [ + { + type: "context", + runtimeBinding: `${makePropSafe("now")}`, + readableBinding: `Date`, + category: "Date", + icon: "Date", + display: { + name: "Server date", + }, + }, + ...getUserBindings(), + ] const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -781,9 +796,8 @@ setRequired(false) } }} - bindings={getBindings({ table })} + bindings={defaultValueBindings} allowJS - context={rowGoldenSample} />
{/if} From f00593ff26ca42e2e1521048aab079b29a3742d3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 1 Oct 2024 12:25:41 +0100 Subject: [PATCH 185/195] pr comments --- .../tests/scenarios/branching.spec.ts | 10 +++++----- .../tests/utilities/AutomationTestBuilder.ts | 20 +++++++++---------- .../server/src/definitions/automations.ts | 4 ++-- packages/server/src/threads/automation.ts | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts index 76e04afdd3..032b729e44 100644 --- a/packages/server/src/automations/tests/scenarios/branching.spec.ts +++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts @@ -30,7 +30,7 @@ describe("Branching automations", () => { .appAction({ fields: {} }) .serverLog( { text: "Starting automation" }, - { stepName: "FirstLog", id: firstLogId } + { stepName: "FirstLog", stepId: firstLogId } ) .branch({ topLevelBranch1: { @@ -38,14 +38,14 @@ describe("Branching automations", () => { stepBuilder .serverLog( { text: "Branch 1" }, - { id: "66666666-6666-6666-6666-666666666666" } + { stepId: "66666666-6666-6666-6666-666666666666" } ) .branch({ branch1: { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.1" }, - { id: branch1LogId } + { stepId: branch1LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: true }, @@ -55,7 +55,7 @@ describe("Branching automations", () => { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.2" }, - { id: branch2LogId } + { stepId: branch2LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, @@ -68,7 +68,7 @@ describe("Branching automations", () => { }, topLevelBranch2: { steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 2" }, { id: branch2Id }), + stepBuilder.serverLog({ text: "Branch 2" }, { stepId: branch2Id }), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, }, diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 6af18cd27e..6aaf22cd6a 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -64,9 +64,9 @@ class BaseStepBuilder { stepId: TStep, stepSchema: Omit, inputs: AutomationStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { - const id = opts?.id || uuidv4() + const id = opts?.stepId || uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, @@ -107,7 +107,7 @@ class BaseStepBuilder { // STEPS createRow( inputs: CreateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.CREATE_ROW, @@ -119,7 +119,7 @@ class BaseStepBuilder { updateRow( inputs: UpdateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.UPDATE_ROW, @@ -131,7 +131,7 @@ class BaseStepBuilder { deleteRow( inputs: DeleteRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.DELETE_ROW, @@ -143,7 +143,7 @@ class BaseStepBuilder { sendSmtpEmail( inputs: SmtpEmailStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, @@ -155,7 +155,7 @@ class BaseStepBuilder { executeQuery( inputs: ExecuteQueryStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, @@ -167,7 +167,7 @@ class BaseStepBuilder { queryRows( inputs: QueryRowsStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.QUERY_ROWS, @@ -178,7 +178,7 @@ class BaseStepBuilder { } loop( inputs: LoopStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.LOOP, @@ -190,7 +190,7 @@ class BaseStepBuilder { serverLog( input: ServerLogStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SERVER_LOG, diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 44758b727b..9433075da7 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,8 +15,8 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] - stepsById?: Record - stepsByName?: Record + stepsById: Record + stepsByName: Record env?: Record trigger: any } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 7788744ea2..3b47634663 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -74,7 +74,7 @@ class Orchestrator { private job: Job private loopStepOutputs: LoopStep[] private stopped: boolean - private executionOutput: AutomationContext + private executionOutput: Omit constructor(job: AutomationJob) { let automation = job.data.automation @@ -458,9 +458,9 @@ class Orchestrator { inputs: steps[stepToLoopIndex].inputs, }) - this.context.stepsById![steps[stepToLoopIndex].id] = tempOutput + this.context.stepsById[steps[stepToLoopIndex].id] = tempOutput const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id - this.context.stepsByName![stepName] = tempOutput + this.context.stepsByName[stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") From ae4f7ae4b45ab4e30778eddff0938017924a22d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:04:01 +0100 Subject: [PATCH 186/195] Implement group by and add a test for it. --- packages/backend-core/src/sql/sql.ts | 19 +++++++++------ .../src/api/routes/tests/viewV2.spec.ts | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 54ca5a0135..14e32623e3 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -799,6 +799,14 @@ class InternalBuilder { return query } + isSqs(t?: Table): boolean { + const table = t || this.table + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) + } + getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { @@ -813,11 +821,7 @@ class InternalBuilder { } let name = table.name - if ( - (table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID) && - table._id - ) { + if (this.isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } @@ -830,7 +834,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as total` ) } @@ -842,10 +846,11 @@ class InternalBuilder { const tableName = this.getTableName() if (fields.length > 0) { query = query.groupBy(fields.map(field => `${tableName}.${field}`)) + query = query.select(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType - const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + const field = `${tableName}.${aggregation.field} as ${aggregation.name}` switch (op) { case CalculationType.COUNT: query = query.count(field) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 0f4e6c961c..b03c445b78 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - // ["lucene", undefined], - // ["sqs", undefined], + ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2459,7 +2459,7 @@ describe.each([ } }) - it.only("should be able to group by a basic field", async () => { + it("should be able to group by a basic field", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -2480,9 +2480,14 @@ describe.each([ query: {}, }) + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + for (const row of response.rows) { - expect(row["quantity"]).toBeGreaterThan(0) - expect(row["Total Price"]).toBeGreaterThan(0) + expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) } }) }) From addd54a8e8baab79811e6f71aab728f32239577b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:39:33 +0100 Subject: [PATCH 187/195] Fix generic-sql.spec.ts --- packages/backend-core/src/sql/sql.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 14e32623e3..0f72eea96e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -811,7 +811,10 @@ class InternalBuilder { let table: Table if (typeof t === "string") { if (!this.query.meta.tables?.[t]) { - throw new Error(`Table ${t} not found`) + // This can legitimately happen in custom queries, where the user is + // querying against a table that may not have been imported into + // Budibase. + return t } table = this.query.meta.tables[t] } else if (t) { @@ -1547,12 +1550,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!this._isJsonColumn(field)) { continue } - const fullName = `${tableName}.${name}` + const fullName = `${tableName}.${name}` as keyof T for (let row of results) { - if (typeof row[fullName as keyof T] === "string") { - row[fullName as keyof T] = JSON.parse(row[fullName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } - if (typeof row[name as keyof T] === "string") { + if (typeof row[name] === "string") { row[name as keyof T] = JSON.parse(row[name]) } } From 7cee1509aa3ee47677e3116121beab3b57a6deef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:17:11 +0100 Subject: [PATCH 188/195] Fix sqlAlias.spec.ts --- packages/backend-core/src/sql/sql.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0f72eea96e..105116e828 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -810,13 +810,18 @@ class InternalBuilder { getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { - if (!this.query.meta.tables?.[t]) { + if (this.query.table?.name === t) { + table = this.query.table + } else if (this.query.meta.table?.name === t) { + table = this.query.meta.table + } else if (!this.query.meta.tables?.[t]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return t + } else { + table = this.query.meta.tables[t] } - table = this.query.meta.tables[t] } else if (t) { table = t } else { From 1dea53f5976aeee63c3fe5f5b8087ac64a42fb90 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 1 Oct 2024 16:25:48 +0100 Subject: [PATCH 189/195] Refresh data when adding columns --- .../DataTable/modals/grid/GridCreateColumnModal.svelte | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte index 2040f66706..d031e752cd 100644 --- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte @@ -2,7 +2,12 @@ import { getContext } from "svelte" import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte" - const { datasource } = getContext("grid") + const { datasource, rows } = getContext("grid") + + const onUpdate = async () => { + await datasource.actions.refreshDefinition() + await rows.actions.refreshData() + } - + From c4a6a92bdbbfee4768c5a77b10aef3c5c8986a43 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:03:06 +0100 Subject: [PATCH 190/195] PR comments --- packages/frontend-core/src/api/ai.js | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/api/ai.js b/packages/frontend-core/src/api/ai.js index 702ce87cdb..7fa756a19e 100644 --- a/packages/frontend-core/src/api/ai.js +++ b/packages/frontend-core/src/api/ai.js @@ -4,7 +4,7 @@ export const buildAIEndpoints = API => ({ */ generateCronExpression: async ({ prompt }) => { return await API.post({ - url: "/api/ai/generate/cron", + url: "/api/ai/cron", body: { prompt }, }) }, diff --git a/packages/pro b/packages/pro index dcc9e50b80..f35190a594 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit dcc9e50b8064a2097d408771462ad80f48de7ff6 +Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 From a28a64f9d8b1053509f58db6acc6ee0c89d92109 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:05:38 +0100 Subject: [PATCH 191/195] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f35190a594..aca9828117 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 +Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7 From 08f1c4dadc668db7fe645f01336832341546828e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:35:15 +0100 Subject: [PATCH 192/195] Update packages/backend-core/src/sql/sql.ts Co-authored-by: Adria Navarro --- packages/backend-core/src/sql/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 105116e828..e55524a67e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1542,7 +1542,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // SQS uses the table ID rather than the table name name = table._id } - return aliases?.[name] ? aliases[name] : name + return aliases?.[name] || name } convertJsonStringColumns>( From ddd229062c20e37f2860c8a07583688f67c0ed1c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:39:54 +0100 Subject: [PATCH 193/195] Rename total field when doing row counts. --- packages/backend-core/src/sql/sql.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e55524a67e..701797329f 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -842,7 +842,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName()}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as __bb_total` ) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d5c0560d9b..3d6bf39d3f 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -57,8 +57,8 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && "total" in response[0]) { - const total = response[0].total + if (response && response.length === 1 && "__bb_total" in response[0]) { + const total = response[0].__bb_total return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From 7b9af81fd510f2aa87c757173abd028c068cebba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:44:20 +0100 Subject: [PATCH 194/195] Clean up params and isSqs --- packages/backend-core/src/sql/sql.ts | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 701797329f..627be039ca 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -87,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +function isSqs(table: Table): boolean { + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -799,37 +806,34 @@ class InternalBuilder { return query } - isSqs(t?: Table): boolean { - const table = t || this.table - return ( - table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID - ) + isSqs(): boolean { + return isSqs(this.table) } - getTableName(t?: Table | string): string { + getTableName(tableOrName?: Table | string): string { let table: Table - if (typeof t === "string") { - if (this.query.table?.name === t) { + if (typeof tableOrName === "string") { + const name = tableOrName + if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === t) { + } else if (this.query.meta.table?.name === name) { table = this.query.meta.table - } else if (!this.query.meta.tables?.[t]) { + } else if (!this.query.meta.tables?.[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. - return t + return name } else { - table = this.query.meta.tables[t] + table = this.query.meta.tables[name] } - } else if (t) { - table = t + } else if (tableOrName) { + table = tableOrName } else { table = this.table } let name = table.name - if (this.isSqs(table) && table._id) { + if (isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } From 61a0db0984ee4bd59094c214ebd55c43e763ce45 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 2 Oct 2024 09:51:13 +0100 Subject: [PATCH 195/195] Add explicit typing for view search filter config --- packages/server/src/sdk/app/rows/search.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 8de5818805..dae24c6bc0 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -3,6 +3,8 @@ import { LogicalOperator, Row, RowSearchParams, + SearchFilter, + SearchFilterGroup, SearchFilterKey, SearchFilters, SearchResponse, @@ -91,11 +93,12 @@ export async function search( if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { // Lucene does not accept conditional filters, so we need to keep the old logic - const query: SearchFilters = viewQuery + const query: SearchFilters = viewQuery || {} + const viewFilters = view.query as SearchFilter[] // Extract existing fields const existingFields = - view.query + viewFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -112,7 +115,7 @@ export async function search( } else { options.query = { $and: { - conditions: [viewQuery, options.query], + conditions: [viewQuery as SearchFilterGroup, options.query], }, } }