From 015ef56110a7c8b06bdc86001d20cd17cb929e07 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 09:29:05 +0100 Subject: [PATCH 001/105] wip --- packages/backend-core/src/sql/sql.ts | 50 +++++++- .../server/src/api/controllers/row/views.ts | 14 ++- .../src/api/controllers/view/viewsV2.ts | 51 ++++++-- .../src/api/routes/tests/viewV2.spec.ts | 71 +++++++++++- packages/server/src/sdk/app/rows/search.ts | 3 + .../src/sdk/app/rows/search/internal/sqs.ts | 23 +++- packages/server/src/sdk/app/views/index.ts | 109 ++++++++++++++---- packages/shared-core/src/helpers/index.ts | 1 + packages/shared-core/src/helpers/schema.ts | 4 + packages/shared-core/src/helpers/views.ts | 39 +++++++ packages/types/src/api/web/app/rows.ts | 1 + packages/types/src/documents/app/view.ts | 18 +-- packages/types/src/sdk/row.ts | 9 +- packages/types/src/sdk/search.ts | 2 + packages/types/src/shared/typeUtils.ts | 23 ++++ 15 files changed, 358 insertions(+), 60 deletions(-) create mode 100644 packages/shared-core/src/helpers/views.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e6738d4b36..b2fec5fe57 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -10,10 +10,12 @@ import { } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { + Aggregation, AnySearchFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, + CalculationType, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -789,6 +791,38 @@ class InternalBuilder { return query.countDistinct(`${aliased}.${primary[0]} as total`) } + addAggregations( + query: Knex.QueryBuilder, + aggregations: Aggregation[] + ): Knex.QueryBuilder { + const fields = this.query.resource?.fields || [] + if (fields.length > 0) { + query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + } + for (const aggregation of aggregations) { + const op = aggregation.calculationType + const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + switch (op) { + case CalculationType.COUNT: + query = query.count(field) + break + case CalculationType.SUM: + query = query.sum(field) + break + case CalculationType.AVG: + query = query.avg(field) + break + case CalculationType.MIN: + query = query.min(field) + break + case CalculationType.MAX: + query = query.max(field) + break + } + } + return query + } + addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort } = this.query const primaryKey = this.table.primary @@ -1172,10 +1206,18 @@ class InternalBuilder { } } - // if counting, use distinct count, else select - query = !counting - ? query.select(this.generateSelectStatement()) - : this.addDistinctCount(query) + const aggregations = this.query.resource?.aggregations || [] + if (counting) { + query = this.addDistinctCount(query) + } else if (aggregations.length > 0) { + query = query.select( + this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") + ) + query = this.addAggregations(query, aggregations) + } else { + query = query.select(this.generateSelectStatement()) + } + // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { query = this.addSorting(query) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index d2541dfa25..ba48bc5664 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,8 +7,9 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + Aggregation, } from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" import sdk from "../../../sdk" import { db, context, features } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" @@ -27,7 +28,7 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } - const viewFields = Object.entries(view.schema || {}) + const viewFields = Object.entries(helpers.views.basicFields(view)) .filter(([_, value]) => value.visible) .map(([key]) => key) const { body } = ctx.request @@ -74,6 +75,14 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) + const aggregations: Aggregation[] = Object.entries( + helpers.views.calculationFields(view) + ).map(([name, { field, calculationType }]) => ({ + name, + calculationType, + field, + })) + const searchOptions: RequiredKeys & RequiredKeys> = { tableId: view.tableId, @@ -84,6 +93,7 @@ export async function searchView( bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, + aggregations, } const result = await sdk.rows.search(searchOptions) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 4208772fa6..18306ba245 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -8,8 +8,45 @@ import { ViewResponse, ViewResponseEnriched, ViewV2, + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" +import { helpers } from "@budibase/shared-core" + +function stripUnknownFields( + field: ViewUIFieldMetadata +): RequiredKeys { + if (helpers.views.isCalculationField(field)) { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + calculationType: field.calculationType, + field: field.field, + } + return strippedField + } else { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + } + return strippedField + } +} + +function stripUndefinedFields(obj: Record): void { + Object.keys(obj) + .filter(key => obj[key] === undefined) + .forEach(key => { + delete obj[key] + }) +} async function parseSchema(view: CreateViewRequest) { if (!view.schema) { @@ -18,18 +55,8 @@ async function parseSchema(view: CreateViewRequest) { const finalViewSchema = view.schema && Object.entries(view.schema).reduce((p, [fieldName, schemaValue]) => { - const fieldSchema: RequiredKeys = { - order: schemaValue.order, - width: schemaValue.width, - visible: schemaValue.visible, - readonly: schemaValue.readonly, - icon: schemaValue.icon, - } - Object.entries(fieldSchema) - .filter(([, val]) => val === undefined) - .forEach(([key]) => { - delete fieldSchema[key as keyof ViewUIFieldMetadata] - }) + const fieldSchema = stripUnknownFields(schemaValue) + stripUndefinedFields(fieldSchema) p[fieldName] = fieldSchema return p }, {} as Record>) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 356f01dee0..72f55c16d2 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -18,6 +18,7 @@ import { ViewV2, SearchResponse, BasicOperator, + CalculationType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -32,13 +33,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1978,6 +1979,64 @@ describe.each([ }) ) }) + + describe("calculations", () => { + let table: Table + let rows: Row[] + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it.only("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) + ) + }) + }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 3d5de2e6cb..95cbc919a1 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -46,6 +46,9 @@ export async function search( paginate: options.paginate, fields: options.fields, countRows: options.countRows, + aggregations: options.aggregations + ?.map(a => `${a.field}:${a.calculationType}`) + .join(", "), }) const isExternalTable = isExternalTableID(options.tableId) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index fb140e3c14..93c6cab2ea 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -48,6 +48,7 @@ import { } from "@budibase/shared-core" import { isSearchingByRowID } from "../utils" import tracer from "dd-trace" +import { cloneDeep } from "lodash" const builder = new sql.Sql(SqlClient.SQL_LITE) const SQLITE_COLUMN_LIMIT = 2000 @@ -285,7 +286,7 @@ export async function search( table: Table, opts?: { retrying?: boolean } ): Promise> { - let { paginate, query, ...params } = options + let { paginate, query, ...params } = cloneDeep(options) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) @@ -303,6 +304,21 @@ export async function search( ...cleanupFilters(query, table, allTables), documentType: DocumentType.ROW, } + + let fields = options.fields + if (fields === undefined) { + fields = buildInternalFieldList(table, allTables, { relationships }) + } else { + fields = fields.map(f => mapToUserColumn(f)) + } + + if (options.aggregations) { + options.aggregations = options.aggregations.map(a => { + a.field = mapToUserColumn(a.field) + return a + }) + } + const request: QueryJson = { endpoint: { // not important, we query ourselves @@ -317,9 +333,7 @@ export async function search( tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, - resource: { - fields: buildInternalFieldList(table, allTables, { relationships }), - }, + resource: { fields, aggregations: options.aggregations }, relationships, } @@ -426,6 +440,7 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } + throw err throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index ed624c2b5c..01c2d3d2fa 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,8 +1,9 @@ import { + BasicViewUIFieldMetadata, RenameColumn, + Table, TableSchema, View, - ViewUIFieldMetadata, ViewV2, ViewV2Enriched, } from "@budibase/types" @@ -38,31 +39,84 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +async function guardCalculationViewSchema( + table: Table, + view: Omit +) { + const calculationFields = helpers.views.calculationFields(view) + for (const calculationFieldName of Object.keys(calculationFields)) { + const schema = calculationFields[calculationFieldName] + const targetSchema = table.schema[schema.field] + if (!targetSchema) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`, + 400 + ) + } + + if (!helpers.schema.isNumeric(targetSchema)) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`, + 400 + ) + } + } + + const groupByFields = helpers.views.basicFields(view) + for (const groupByFieldName of Object.keys(groupByFields)) { + const targetSchema = table.schema[groupByFieldName] + if (!targetSchema) { + throw new HTTPError( + `Group by field "${groupByFieldName}" does not exist in the table schema`, + 400 + ) + } + } +} + async function guardViewSchema( tableId: string, view: Omit ) { - const viewSchema = view.schema || {} const table = await sdk.tables.getTable(tableId) + if (helpers.views.isCalculationView(view)) { + await guardCalculationViewSchema(table, view) + } + + await checkReadonlyFields(table, view) + checkRequiredFields(table, view) + checkDisplayField(view) +} + +async function checkReadonlyFields( + table: Table, + view: Omit +) { + const viewSchema = view.schema || {} for (const field of Object.keys(viewSchema)) { - const tableSchemaField = table.schema[field] - if (!tableSchemaField) { + const viewFieldSchema = viewSchema[field] + if (helpers.views.isCalculationField(viewFieldSchema)) { + continue + } + + const tableFieldSchema = table.schema[field] + if (!tableFieldSchema) { throw new HTTPError( `Field "${field}" is not valid for the requested table`, 400 ) } - if (viewSchema[field].readonly) { + if (viewFieldSchema.readonly) { if ( !(await features.isViewReadonlyColumnsEnabled()) && - !(tableSchemaField as ViewUIFieldMetadata).readonly + !(tableFieldSchema as BasicViewUIFieldMetadata).readonly ) { throw new HTTPError(`Readonly fields are not enabled`, 400) } - if (!viewSchema[field].visible) { + if (!viewFieldSchema.visible) { throw new HTTPError( `Field "${field}" must be visible if you want to make it readonly`, 400 @@ -70,18 +124,33 @@ async function guardViewSchema( } } } +} - const existingView = - table?.views && (table.views[view.name] as ViewV2 | undefined) +function checkDisplayField(view: Omit) { + if (view.primaryDisplay) { + const viewSchemaField = view.schema?.[view.primaryDisplay] + if (!viewSchemaField?.visible) { + throw new HTTPError( + `You can't hide "${view.primaryDisplay}" because it is the display column.`, + 400 + ) + } + } +} + +function checkRequiredFields( + table: Table, + view: Omit +) { + const existingView = table.views?.[view.name] as ViewV2 | undefined for (const field of Object.values(table.schema)) { if (!helpers.schema.isRequired(field.constraints)) { continue } - const viewSchemaField = viewSchema[field.name] - const existingViewSchema = - existingView?.schema && existingView.schema[field.name] + const viewSchemaField = view.schema?.[field.name] + const existingViewSchema = existingView?.schema?.[field.name] if (!viewSchemaField && !existingViewSchema?.visible) { // Supporting existing configs with required columns but hidden in views continue @@ -94,24 +163,16 @@ async function guardViewSchema( ) } - if (viewSchemaField.readonly) { + if ( + helpers.views.isBasicViewField(viewSchemaField) && + viewSchemaField.readonly + ) { throw new HTTPError( `You can't make "${field.name}" readonly because it is a required field.`, 400 ) } } - - if (view.primaryDisplay) { - const viewSchemaField = viewSchema[view.primaryDisplay] - - if (!viewSchemaField?.visible) { - throw new HTTPError( - `You can't hide "${view.primaryDisplay}" because it is the display column.`, - 400 - ) - } - } } export async function create( diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts index c33ff2ab1e..503f71e4eb 100644 --- a/packages/shared-core/src/helpers/index.ts +++ b/packages/shared-core/src/helpers/index.ts @@ -2,3 +2,4 @@ export * from "./helpers" export * from "./integrations" export * as cron from "./cron" export * as schema from "./schema" +export * as views from "./views" diff --git a/packages/shared-core/src/helpers/schema.ts b/packages/shared-core/src/helpers/schema.ts index d0035cc305..c9200fee18 100644 --- a/packages/shared-core/src/helpers/schema.ts +++ b/packages/shared-core/src/helpers/schema.ts @@ -45,3 +45,7 @@ export function decodeNonAscii(str: string): string { String.fromCharCode(parseInt(p1, 16)) ) } + +export function isNumeric(field: FieldSchema) { + return field.type === FieldType.NUMBER || field.type === FieldType.BIGINT +} diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts new file mode 100644 index 0000000000..0364ccff41 --- /dev/null +++ b/packages/shared-core/src/helpers/views.ts @@ -0,0 +1,39 @@ +import { + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, + ViewUIFieldMetadata, + ViewV2, +} from "@budibase/types" +import { pickBy } from "lodash" + +export function isCalculationField( + field: ViewUIFieldMetadata +): field is ViewCalculationFieldMetadata { + return "calculationType" in field +} + +export function isBasicViewField( + field: ViewUIFieldMetadata +): field is BasicViewUIFieldMetadata { + return !isCalculationField(field) +} + +type UnsavedViewV2 = Omit + +export function isCalculationView(view: UnsavedViewV2) { + return Object.values(view.schema || {}).some(isCalculationField) +} + +export function calculationFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, isCalculationField) +} + +export function basicFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, field => !isCalculationField(field)) +} diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index ce6f6f672d..5d49ac1812 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -26,6 +26,7 @@ export interface SearchViewRowRequest | "paginate" | "query" | "countRows" + | "aggregations" > {} export interface SearchRowResponse { diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 24dad0bcca..539c1e0d23 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -33,10 +33,19 @@ export interface View { groupBy?: string } -export type ViewUIFieldMetadata = UIFieldMetadata & { +export interface BasicViewUIFieldMetadata extends UIFieldMetadata { readonly?: boolean } +export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata { + calculationType: CalculationType + field: string +} + +export type ViewUIFieldMetadata = + | BasicViewUIFieldMetadata + | ViewCalculationFieldMetadata + export enum CalculationType { SUM = "sum", AVG = "avg", @@ -45,11 +54,6 @@ export enum CalculationType { MAX = "max", } -export type ViewCalculationFieldMetadata = ViewUIFieldMetadata & { - calculationType: CalculationType - field: string -} - export interface ViewV2 { version: 2 id: string @@ -62,7 +66,7 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: Record } export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 6850359cc3..8ee0338731 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -1,8 +1,14 @@ import { SortOrder, SortType } from "../api" import { SearchFilters } from "./search" -import { Row } from "../documents" +import { CalculationType, Row } from "../documents" import { WithRequired } from "../shared" +export interface Aggregation { + name: string + calculationType: CalculationType + field: string +} + export interface SearchParams { tableId?: string query?: SearchFilters @@ -18,6 +24,7 @@ export interface SearchParams { indexer?: () => Promise rows?: Row[] countRows?: boolean + aggregations?: Aggregation[] } // when searching for rows we want a more extensive search type that requires certain properties diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..ba3c388480 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -2,6 +2,7 @@ import { Operation } from "./datasources" import { Row, Table, DocumentType } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" +import { Aggregation } from "./row" export enum BasicOperator { EQUAL = "equal", @@ -143,6 +144,7 @@ export interface QueryJson { } resource?: { fields: string[] + aggregations?: Aggregation[] } filters?: SearchFilters sort?: SortJson diff --git a/packages/types/src/shared/typeUtils.ts b/packages/types/src/shared/typeUtils.ts index c7ecebed0a..dbb3fc2553 100644 --- a/packages/types/src/shared/typeUtils.ts +++ b/packages/types/src/shared/typeUtils.ts @@ -4,6 +4,29 @@ export type DeepPartial = { export type ISO8601 = string +/** + * RequiredKeys make it such that you _must_ assign a value to every key in the + * type. It differs subtly from Required in that it doesn't change the type + * of the fields, you can specify undefined as a value and that's fine. + * + * Example: + * + * ```ts + * interface Foo { + * bar: string + * baz?: string + * } + * + * type FooRequiredKeys = RequiredKeys + * type FooRequired = Required + * + * const a: FooRequiredKeys = { bar: "hello", baz: undefined } + * const b: FooRequired = { bar: "hello", baz: undefined } + * ``` + * + * In this code, a passes type checking whereas b does not. This is because + * Required makes baz non-optional. + */ export type RequiredKeys = { [K in keyof Required]: T[K] } From fc44b38fc5e5a2cab4a920e4cf0aac6d6d855247 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 11:52:06 +0100 Subject: [PATCH 002/105] Fix tests. --- .../server/src/api/routes/tests/viewV2.spec.ts | 16 ++++++++-------- packages/shared-core/src/helpers/views.ts | 6 ------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 6769c80867..9313e3ab4d 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,13 +37,13 @@ import { import sdk from "../../../sdk" describe.each([ - // ["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2295,7 +2295,7 @@ describe.each([ ) }) - describe("calculations", () => { + describe.skip("calculations", () => { let table: Table let rows: Row[] @@ -2325,7 +2325,7 @@ describe.each([ ) }) - it.only("should be able to search by calculations", async () => { + it("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts index 0364ccff41..c65bc4882d 100644 --- a/packages/shared-core/src/helpers/views.ts +++ b/packages/shared-core/src/helpers/views.ts @@ -25,15 +25,9 @@ export function isCalculationView(view: UnsavedViewV2) { } export function calculationFields(view: UnsavedViewV2) { - if (!isCalculationView(view)) { - throw new Error("View is not a calculation view") - } return pickBy(view.schema || {}, isCalculationField) } export function basicFields(view: UnsavedViewV2) { - if (!isCalculationView(view)) { - throw new Error("View is not a calculation view") - } return pickBy(view.schema || {}, field => !isCalculationField(field)) } From c5db1d1da305e350804685e59877780aafa45011 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 16:37:23 +0100 Subject: [PATCH 003/105] Got a test passing but I hate it a bit. --- .../src/api/controllers/row/utils/basic.ts | 16 +++++++++++++++- .../src/api/controllers/row/utils/utils.ts | 4 +++- .../server/src/api/routes/tests/viewV2.spec.ts | 16 ++++++++-------- packages/server/src/db/linkedRows/index.ts | 16 ++++++++++++---- packages/server/src/sdk/app/rows/search.ts | 1 + .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++++++--- .../server/src/utilities/rowProcessor/index.ts | 9 +++++++++ 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9c49f5636a..20a1cd0742 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,12 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types" +import { + FieldSchema, + FieldType, + Row, + Table, + JsonTypes, + Aggregation, +} from "@budibase/types" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -84,12 +91,14 @@ export function basicProcessing({ tables, isLinked, sqs, + aggregations, }: { row: Row table: Table tables: Table[] isLinked: boolean sqs?: boolean + aggregations?: Aggregation[] }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) @@ -108,6 +117,11 @@ export function basicProcessing({ thisRow[fieldName] = value } } + + for (let aggregation of aggregations || []) { + thisRow[aggregation.name] = row[aggregation.name] + } + let columns: string[] = Object.keys(table.schema) if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index ac305e70b6..1fc7221294 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -2,6 +2,7 @@ import * as utils from "../../../../db/utils" import { context } from "@budibase/backend-core" import { + Aggregation, Ctx, DatasourcePlusQueryResponse, FieldType, @@ -129,7 +130,7 @@ export async function sqlOutputProcessing( table: Table, tables: Record, relationships: RelationshipsJson[], - opts?: { sqs?: boolean } + opts?: { sqs?: boolean; aggregations?: Aggregation[] } ): Promise { if (isKnexEmptyReadResponse(rows)) { return [] @@ -150,6 +151,7 @@ export async function sqlOutputProcessing( tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, + aggregations: opts?.aggregations, }) if (thisRow._id == null) { throw new Error("Unable to generate row ID for SQL rows") diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aaaf03c113..dc940c5ace 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,13 +37,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2215,7 +2215,7 @@ describe.each([ ) }) - describe.skip("calculations", () => { + describe("calculations", () => { let table: Table let rows: Row[] @@ -2245,7 +2245,7 @@ describe.each([ ) }) - it("should be able to search by calculations", async () => { + it.only("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..4222f9b5e4 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -20,10 +20,11 @@ import { Row, Table, TableSchema, - ViewFieldMetadata, + ViewUIFieldMetadata, ViewV2, } from "@budibase/types" import sdk from "../../sdk" +import { helpers } from "@budibase/shared-core" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -264,12 +265,19 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} - if (options?.fromViewId && allowRelationshipSchemas) { + let viewSchema: Record = {} + if (options?.fromViewId) { const view = Object.values(table.views || {}).find( (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId ) - viewSchema = view?.schema || {} + + if (view && helpers.views.isCalculationView(view)) { + return enriched + } + + if (allowRelationshipSchemas && view) { + viewSchema = view.schema || {} + } } // will populate this as we find them diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 95cbc919a1..dbf0cefd51 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -37,6 +37,7 @@ export async function search( return await tracer.trace("search", async span => { span?.addTags({ tableId: options.tableId, + viewId: options.viewId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 188c95fb5c..b5bf8e752f 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -386,8 +386,9 @@ export async function search( // make sure JSON columns corrected const processed = builder.convertJsonStringColumns( table, - await sqlOutputProcessing(rows, table!, allTablesMap, relationships, { + await sqlOutputProcessing(rows, table, allTablesMap, relationships, { sqs: true, + aggregations: options.aggregations, }) ) @@ -406,11 +407,16 @@ export async function search( preserveLinks: true, squash: true, fromViewId: options.viewId, + aggregations: options.aggregations, }) // check if we need to pick specific rows out if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] + const fields = [ + ...options.fields, + ...PROTECTED_INTERNAL_COLUMNS, + ...(options.aggregations || []).map(a => a.name), + ] finalRows = finalRows.map((r: any) => pick(r, fields)) } @@ -440,6 +446,5 @@ export async function search( return { rows: [] } } throw err - throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f6cf44d6d6..73768fdd57 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,6 +11,7 @@ import { import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { + Aggregation, AutoFieldSubType, FieldType, IdentityType, @@ -250,6 +251,7 @@ export async function outputProcessing( fromRow?: Row skipBBReferences?: boolean fromViewId?: string + aggregations?: Aggregation[] } = { squash: true, preserveLinks: false, @@ -357,6 +359,7 @@ export async function outputProcessing( fromViewId: opts?.fromViewId, }) } + // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) if (isExternal || (await features.flags.isEnabled("SQS"))) { @@ -385,9 +388,15 @@ export async function outputProcessing( const tableFields = Object.keys(table.schema).filter( f => table.schema[f].visible !== false ) + const fields = [...tableFields, ...protectedColumns].map(f => f.toLowerCase() ) + + for (const aggregation of opts.aggregations || []) { + fields.push(aggregation.name.toLowerCase()) + } + for (const row of enriched) { for (const key of Object.keys(row)) { if (!fields.includes(key.toLowerCase())) { From 51774b3434a344e0ab8e766b96ad2e840ce44033 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 12:30:45 +0100 Subject: [PATCH 004/105] Working on plumbing 'source' all the way through our code. --- .../backend-core/src/context/mainContext.ts | 19 +++- packages/backend-core/src/context/types.ts | 3 +- packages/backend-core/src/docIds/params.ts | 14 +-- packages/backend-core/src/sql/sql.ts | 3 - .../api/controllers/row/ExternalRequest.ts | 89 ++++++++++--------- .../src/api/controllers/row/external.ts | 19 ++-- .../src/api/controllers/row/utils/utils.ts | 13 ++- .../server/src/api/controllers/row/views.ts | 7 +- packages/server/src/db/utils.ts | 8 +- .../middleware/triggerRowActionAuthorised.ts | 2 +- .../server/src/sdk/app/permissions/index.ts | 6 +- packages/server/src/sdk/app/rowActions.ts | 6 +- .../server/src/sdk/app/rows/queryUtils.ts | 7 +- packages/server/src/sdk/app/rows/rows.ts | 5 +- packages/server/src/sdk/app/rows/search.ts | 58 +++++++----- .../src/sdk/app/rows/search/external.ts | 7 +- .../server/src/sdk/app/rows/search/utils.ts | 6 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 20 ++--- packages/server/src/sdk/app/rows/utils.ts | 35 +++----- packages/server/src/sdk/app/tables/utils.ts | 4 + packages/server/src/sdk/app/views/index.ts | 19 +++- packages/types/src/sdk/row.ts | 5 +- yarn.lock | 64 +------------ 23 files changed, 204 insertions(+), 215 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index a52a17dd53..25b273e51c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext, Snippet, App } from "@budibase/types" +import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record) { context.featureFlagCache ??= {} context.featureFlagCache[key] = value } + +export function getTableForView(viewId: string): Table | undefined { + const context = getCurrentContext() + if (!context) { + return + } + return context.viewToTableCache?.[viewId] +} + +export function setTableForView(viewId: string, table: Table) { + const context = getCurrentContext() + if (!context) { + return + } + context.viewToTableCache ??= {} + context.viewToTableCache[viewId] = table +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index fe6072e85c..ee84b49459 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, Snippet, VM } from "@budibase/types" +import { IdentityContext, Snippet, Table, VM } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { GoogleSpreadsheet } from "google-spreadsheet" @@ -21,4 +21,5 @@ export type ContextMap = { featureFlagCache?: { [key: string]: Record } + viewToTableCache?: Record } diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 093724b55e..016604b69b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,7 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" -import { DatabaseQueryOpts } from "@budibase/types" +import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) { /** * Check if a given ID is that of a table. - * @returns {boolean} */ -export const isTableId = (id: string) => { +export const isTableId = (id: string): boolean => { // this includes datasource plus tables return ( !!id && @@ -77,13 +76,16 @@ export const isTableId = (id: string) => { ) } +export function isViewId(id: string): boolean { + return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`) +} + /** * Check if a given ID is that of a datasource or datasource plus. - * @returns {boolean} */ -export const isDatasourceId = (id: string) => { +export const isDatasourceId = (id: string): boolean => { // this covers both datasources and datasource plus - return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) + return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) } /** diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 023bfd5d8a..97fc4124fb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1274,9 +1274,6 @@ class InternalBuilder { if (counting) { query = this.addDistinctCount(query) } else if (aggregations.length > 0) { - query = query.select( - this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") - ) query = this.addAggregations(query, aggregations) } else { query = query.select(this.generateSelectStatement()) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 9c9bd0b284..a52d7abcd1 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -19,6 +19,7 @@ import { SortJson, SortType, Table, + ViewV2, } from "@budibase/types" import { breakExternalTableId, @@ -159,17 +160,41 @@ function isEditableColumn(column: FieldSchema) { export class ExternalRequest { private readonly operation: T - private readonly tableId: string - private datasource?: Datasource - private tables: { [key: string]: Table } = {} + private readonly source: Table | ViewV2 + private datasource: Datasource - constructor(operation: T, tableId: string, datasource?: Datasource) { - this.operation = operation - this.tableId = tableId - this.datasource = datasource - if (datasource && datasource.entities) { - this.tables = datasource.entities + public static async for( + operation: T, + source: Table | ViewV2, + opts: { datasource?: Datasource } = {} + ) { + if (!opts.datasource) { + if (sdk.views.isView(source)) { + const table = await sdk.views.getTable(source.id) + opts.datasource = await sdk.datasources.get(table.sourceId!) + } else { + opts.datasource = await sdk.datasources.get(source.sourceId!) + } } + + return new ExternalRequest(operation, source, opts.datasource) + } + + private get tables(): { [key: string]: Table } { + if (!this.datasource.entities) { + throw new Error("Datasource does not have entities") + } + return this.datasource.entities + } + + private constructor( + operation: T, + source: Table | ViewV2, + datasource: Datasource + ) { + this.operation = operation + this.source = source + this.datasource = datasource } private prepareFilters( @@ -290,20 +315,6 @@ export class ExternalRequest { return this.tables[tableName] } - // seeds the object with table and datasource information - async retrieveMetadata( - datasourceId: string - ): Promise<{ tables: Record; datasource: Datasource }> { - if (!this.datasource) { - this.datasource = await sdk.datasources.get(datasourceId) - if (!this.datasource || !this.datasource.entities) { - throw "No tables found, fetch tables before query." - } - this.tables = this.datasource.entities - } - return { tables: this.tables, datasource: this.datasource } - } - async getRow(table: Table, rowId: string): Promise { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), @@ -619,24 +630,16 @@ export class ExternalRequest { } async run(config: RunConfig): Promise> { - const { operation, tableId } = this - if (!tableId) { - throw new Error("Unable to run without a table ID") - } - let { datasourceId, tableName } = breakExternalTableId(tableId) - let datasource = this.datasource - if (!datasource) { - const { datasource: ds } = await this.retrieveMetadata(datasourceId) - datasource = ds - } - const tables = this.tables - const table = tables[tableName] - let isSql = isSQL(datasource) - if (!table) { - throw new Error( - `Unable to process query, table "${tableName}" not defined.` - ) + const { operation } = this + let table: Table + if (sdk.views.isView(this.source)) { + table = await sdk.views.getTable(this.source.id) + } else { + table = this.source } + + let isSql = isSQL(this.datasource) + // look for specific components of config which may not be considered acceptable let { id, row, filters, sort, paginate, rows } = cleanupConfig( config, @@ -687,8 +690,8 @@ export class ExternalRequest { } let json: QueryJson = { endpoint: { - datasourceId: datasourceId!, - entityId: tableName, + datasourceId: this.datasource._id!, + entityId: table.name, operation, }, resource: { @@ -714,7 +717,7 @@ export class ExternalRequest { }, meta: { table, - tables: tables, + tables: this.tables, }, } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index bd5201c05c..fb992059f4 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -17,6 +17,7 @@ import { Row, Table, UserCtx, + ViewV2, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" @@ -29,29 +30,29 @@ import { generateIdForRow } from "./utils" export async function handleRequest( operation: T, - tableId: string, + source: Table | ViewV2, opts?: RunConfig ): Promise> { - return new ExternalRequest(operation, tableId, opts?.datasource).run( - opts || {} - ) + return ( + await ExternalRequest.for(operation, source, { + datasource: opts?.datasource, + }) + ).run(opts || {}) } export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) - + const source = await utils.getSource(ctx) const { _id, ...rowData } = ctx.request.body - const table = await sdk.tables.getTable(tableId) const { row: dataToUpdate } = await inputProcessing( ctx.user?._id, - cloneDeep(table), + cloneDeep(source), rowData ) const validateResult = await sdk.rows.utils.validate({ row: dataToUpdate, - tableId, + source, }) if (!validateResult.valid) { throw { validation: validateResult.errors } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 1fc7221294..91c0fc966f 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context } from "@budibase/backend-core" +import { context, docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -9,6 +9,7 @@ import { RelationshipsJson, Row, Table, + ViewV2, } from "@budibase/types" import { processDates, @@ -78,7 +79,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { const { sourceId } = ctx.params - if (utils.isViewID(sourceId)) { + if (docIds.isViewId(sourceId)) { return { tableId: utils.extractViewInfoFromID(sourceId).tableId, viewId: sourceId, @@ -97,6 +98,14 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { throw new Error("Unable to find table ID in request") } +export async function getSource(ctx: Ctx): Promise { + const { tableId, viewId } = getSourceId(ctx) + if (viewId) { + return sdk.views.get(viewId) + } + return sdk.tables.getTable(tableId) +} + export async function validate( opts: { row: Row } & ({ tableId: string } | { table: Table }) ) { diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f9565f0e82..7008c5e0be 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -84,11 +84,8 @@ export async function searchView( })) const searchOptions: RequiredKeys & - RequiredKeys< - Pick - > = { - tableId: view.tableId, - viewId: view.id, + RequiredKeys> = { + sourceId: view.id, query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 043394e7a6..6c1065e847 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,4 +1,4 @@ -import { context, db as dbCore, utils } from "@budibase/backend-core" +import { context, db as dbCore, docIds, utils } from "@budibase/backend-core" import { DatabaseQueryOpts, Datasource, @@ -318,12 +318,8 @@ export function generateViewID(tableId: string) { }${SEPARATOR}${tableId}${SEPARATOR}${newid()}` } -export function isViewID(viewId: string) { - return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW -} - export function extractViewInfoFromID(viewId: string) { - if (!isViewID(viewId)) { + if (!docIds.isViewId(viewId)) { throw new Error("Unable to extract table ID, is not a view ID") } const split = viewId.split(SEPARATOR) diff --git a/packages/server/src/middleware/triggerRowActionAuthorised.ts b/packages/server/src/middleware/triggerRowActionAuthorised.ts index 17f22b7000..3e903ba907 100644 --- a/packages/server/src/middleware/triggerRowActionAuthorised.ts +++ b/packages/server/src/middleware/triggerRowActionAuthorised.ts @@ -15,7 +15,7 @@ export function triggerRowActionAuthorised( const rowActionId: string = ctx.params[actionPath] const isTableId = docIds.isTableId(sourceId) - const isViewId = utils.isViewID(sourceId) + const isViewId = docIds.isViewId(sourceId) if (!isTableId && !isViewId) { ctx.throw(400, `'${sourceId}' is not a valid source id`) } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index a6e81652ee..d5e4aefe3a 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,10 +1,10 @@ -import { db, roles } from "@budibase/backend-core" +import { db, docIds, roles } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, } from "@budibase/types" -import { extractViewInfoFromID, isViewID } from "../../../db/utils" +import { extractViewInfoFromID } from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, @@ -20,7 +20,7 @@ type ResourcePermissions = Record< export async function getInheritablePermissions( resourceId: string ): Promise { - if (isViewID(resourceId)) { + if (docIds.isViewId(resourceId)) { return await getResourcePerms(extractViewInfoFromID(resourceId).tableId) } } diff --git a/packages/server/src/sdk/app/rowActions.ts b/packages/server/src/sdk/app/rowActions.ts index 9a7d402df0..21c256eacb 100644 --- a/packages/server/src/sdk/app/rowActions.ts +++ b/packages/server/src/sdk/app/rowActions.ts @@ -1,11 +1,11 @@ -import { context, HTTPError, utils } from "@budibase/backend-core" +import { context, docIds, HTTPError, utils } from "@budibase/backend-core" import { AutomationTriggerStepId, SEPARATOR, TableRowActions, VirtualDocumentType, } from "@budibase/types" -import { generateRowActionsID, isViewID } from "../../db/utils" +import { generateRowActionsID } from "../../db/utils" import automations from "./automations" import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo" import * as triggers from "../../automations/triggers" @@ -155,7 +155,7 @@ export async function update( async function guardView(tableId: string, viewId: string) { let view - if (isViewID(viewId)) { + if (docIds.isViewId(viewId)) { view = await sdk.views.get(viewId) } if (!view || view.tableId !== tableId) { diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 65f400a1d9..a88763215e 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -53,8 +53,8 @@ export const removeInvalidFilters = ( } export const getQueryableFields = async ( - fields: string[], - table: Table + table: Table, + fields?: string[] ): Promise => { const extractTableFields = async ( table: Table, @@ -110,6 +110,9 @@ export const getQueryableFields = async ( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] + if (fields === undefined) { + fields = Object.keys(table.schema) + } result.push(...(await extractTableFields(table, fields, [table._id!]))) return result diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index c61b8692ed..ef25d06baf 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,9 +1,8 @@ -import { db as dbCore, context } from "@budibase/backend-core" +import { db as dbCore, context, docIds } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" import { extractViewInfoFromID, getRowParams, - isViewID, } from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" @@ -26,7 +25,7 @@ export async function getAllInternalRows(appId?: string) { function pickApi(tableOrViewId: string) { let tableId = tableOrViewId - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { tableId = extractViewInfoFromID(tableOrViewId).tableId } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index dbf0cefd51..fa01a6cf13 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -4,6 +4,8 @@ import { RowSearchParams, SearchResponse, SortOrder, + Table, + ViewV2, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -12,7 +14,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features } from "@budibase/backend-core" +import { features, docIds } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" @@ -36,8 +38,7 @@ export async function search( ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ - tableId: options.tableId, - viewId: options.viewId, + sourceId: options.sourceId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, @@ -52,20 +53,18 @@ export async function search( .join(", "), }) - const isExternalTable = isExternalTableID(options.tableId) options.query = dataFilters.cleanupQuery(options.query || {}) options.query = dataFilters.fixupFilterArrays(options.query) - span?.addTags({ + span.addTags({ cleanedQuery: options.query, - isExternalTable, }) if ( !dataFilters.hasFilters(options.query) && options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE ) { - span?.addTags({ emptyQuery: true }) + span.addTags({ emptyQuery: true }) return { rows: [], } @@ -75,34 +74,47 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - const table = await sdk.tables.getTable(options.tableId) - options = searchInputMapping(table, options) + let source: Table | ViewV2 + let table: Table + if (docIds.isTableId(options.sourceId)) { + source = await sdk.tables.getTable(options.sourceId) + table = source + options = searchInputMapping(source, options) + } else if (docIds.isViewId(options.sourceId)) { + source = await sdk.views.get(options.sourceId) + table = await sdk.tables.getTable(source.tableId) + options = searchInputMapping(table, options) - if (options.query) { - const tableFields = Object.keys(table.schema).filter( - f => table.schema[f].visible !== false - ) - - const queriableFields = await getQueryableFields( - options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, - table - ) - options.query = removeInvalidFilters(options.query, queriableFields) + span.addTags({ + tableId: table._id, + }) + } else { + throw new Error(`Invalid source ID: ${options.sourceId}`) } + if (options.query) { + const visibleFields = ( + options.fields || Object.keys(table.schema) + ).filter(field => table.schema[field].visible) + + const queryableFields = await getQueryableFields(table, visibleFields) + options.query = removeInvalidFilters(options.query, queryableFields) + } + + const isExternalTable = isExternalTableID(table._id!) let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) - result = await external.search(options, table) + result = await external.search(options, source) } else if (await features.flags.isEnabled("SQS")) { span?.addTags({ searchType: "sqs" }) - result = await internal.sqs.search(options, table) + result = await internal.sqs.search(options, source) } else { span?.addTags({ searchType: "lucene" }) - result = await internal.lucene.search(options, table) + result = await internal.lucene.search(options, source) } - span?.addTags({ + span.addTags({ foundRows: result.rows.length, totalRows: result.totalRows, }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 0ff25a00e4..5584b3f110 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -9,6 +9,7 @@ import { SortJson, SortOrder, Table, + ViewV2, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -60,9 +61,8 @@ function getPaginationAndLimitParameters( export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = @@ -112,10 +112,9 @@ export async function search( : Promise.resolve(undefined), ]) - let processed = await outputProcessing(table, rows, { + let processed = await outputProcessing(source, rows, { preserveLinks: true, squash: true, - fromViewId: options.viewId, }) let hasNextPage = false diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 5ffc065353..d727e58887 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -9,6 +9,7 @@ import { SearchResponse, Row, RowSearchParams, + ViewV2, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" @@ -83,10 +84,7 @@ function userColumnMapping(column: string, options: RowSearchParams) { // maps through the search parameters to check if any of the inputs are invalid // based on the table schema, converts them to something that is valid. export function searchInputMapping(table: Table, options: RowSearchParams) { - if (!table?.schema) { - return options - } - for (let [key, column] of Object.entries(table.schema)) { + for (let [key, column] of Object.entries(table.schema || {})) { switch (column.type) { case FieldType.BB_REFERENCE_SINGLE: { const subtype = column.subtype diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index aabc359484..f399801f1e 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -203,7 +203,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name", "age"]) }) @@ -216,7 +216,7 @@ describe("query utils", () => { }, }) - const result = await getQueryableFields(Object.keys(table.schema), table) + const result = await getQueryableFields(table) expect(result).toEqual(["_id", "name"]) }) @@ -245,7 +245,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -282,7 +282,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) }) @@ -313,7 +313,7 @@ describe("query utils", () => { }) const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual(["_id", "name"]) }) @@ -381,7 +381,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -398,7 +398,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 1 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux1.schema), aux1) + return getQueryableFields(aux1) }) expect(result).toEqual([ "_id", @@ -420,7 +420,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux 2 table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux2.schema), aux2) + return getQueryableFields(aux2) }) expect(result).toEqual([ "_id", @@ -474,7 +474,7 @@ describe("query utils", () => { it("includes nested relationship fields from main table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + return getQueryableFields(table) }) expect(result).toEqual([ "_id", @@ -488,7 +488,7 @@ describe("query utils", () => { it("includes nested relationship fields from aux table", async () => { const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux.schema), aux) + return getQueryableFields(aux) }) expect(result).toEqual([ "_id", diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bc09116b3b..3899009f13 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -13,16 +13,14 @@ import { TableSchema, SqlClient, ArrayOperator, + ViewV2, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." -import { - extractViewInfoFromID, - isRelationshipColumn, - isViewID, -} from "../../../db/utils" +import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" +import { docIds } from "@budibase/backend-core" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -142,37 +140,32 @@ function isForeignKey(key: string, table: Table) { } export async function validate({ - tableId, + source, row, - table, }: { - tableId?: string + source: Table | ViewV2 row: Row - table?: Table }): Promise<{ valid: boolean errors: Record }> { - let fetchedTable: Table | undefined - if (!table && tableId) { - fetchedTable = await sdk.tables.getTable(tableId) - } else if (table) { - fetchedTable = table - } - if (fetchedTable === undefined) { - throw new Error("Unable to fetch table for validation") + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source } const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, FieldType.BB_REFERENCE_SINGLE, ] - for (let fieldName of Object.keys(fetchedTable.schema)) { - const column = fetchedTable.schema[fieldName] + for (let fieldName of Object.keys(table.schema)) { + const column = table.schema[fieldName] const constraints = cloneDeep(column.constraints) const type = column.type // foreign keys are likely to be enriched - if (isForeignKey(fieldName, fetchedTable)) { + if (isForeignKey(fieldName, table)) { continue } // formulas shouldn't validated, data will be deleted anyway @@ -323,7 +316,7 @@ export function isArrayFilter(operator: any): operator is ArrayOperator { } export function tryExtractingTableAndViewId(tableOrViewId: string) { - if (isViewID(tableOrViewId)) { + if (docIds.isViewId(tableOrViewId)) { return { tableId: extractViewInfoFromID(tableOrViewId).tableId, viewId: tableOrViewId, diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index b8e3d888af..7a8096fb0a 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -9,3 +9,7 @@ export function isExternal(opts: { table?: Table; tableId?: string }): boolean { } return false } + +export function isTable(table: any): table is Table { + return table.type === "table" +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index de3579f7fd..f911385dc1 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -9,7 +9,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -40,6 +40,23 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +export async function getTable(viewId: string): Promise
{ + const cached = context.getTableForView(viewId) + if (cached) { + return cached + } + const { tableId } = utils.extractViewInfoFromID(viewId) + const table = await sdk.tables.getTable(tableId) + context.setTableForView(viewId, table) + return table +} + +export function isView(view: any): view is ViewV2 { + return ( + view.version === 2 && "id" in view && "tableId" in view && "name" in view + ) +} + async function guardCalculationViewSchema( table: Table, view: Omit diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index f81d56c082..2b6ff3a6c6 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -10,8 +10,7 @@ export interface Aggregation { } export interface SearchParams { - tableId?: string - viewId?: string + sourceId?: string query?: SearchFilters paginate?: boolean bookmark?: string | number @@ -30,7 +29,7 @@ export interface SearchParams { // when searching for rows we want a more extensive search type that requires certain properties export interface RowSearchParams - extends WithRequired {} + extends WithRequired {} export interface SearchResponse { rows: T[] diff --git a/yarn.lock b/yarn.lock index cd850e833d..01466f66b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17796,21 +17796,11 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" -pg-cloudflare@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" - integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== - pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== -pg-connection-string@^2.6.4: - version "2.6.4" - resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d" - integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA== - pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17821,21 +17811,11 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== -pg-pool@^3.6.2: - version "3.6.2" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2" - integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg== - pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== -pg-protocol@^1.6.1: - version "1.6.1" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" - integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== - pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17860,19 +17840,6 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" -pg@^8.12.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79" - integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ== - dependencies: - pg-connection-string "^2.6.4" - pg-pool "^3.6.2" - pg-protocol "^1.6.1" - pg-types "^2.1.0" - pgpass "1.x" - optionalDependencies: - pg-cloudflare "^1.1.1" - pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20786,16 +20753,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20886,7 +20844,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20900,13 +20858,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22862,7 +22813,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22880,15 +22831,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From fc9b54cb858c743f0530947b7aa984e974e6ceb9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 13:01:33 +0100 Subject: [PATCH 005/105] Mostly solving type errors around passing the view all the way down, got a fair few left. --- .../src/api/controllers/row/external.ts | 21 +++--- .../server/src/api/controllers/row/index.ts | 9 +-- .../src/api/controllers/row/utils/utils.ts | 16 ++--- packages/server/src/sdk/app/rows/external.ts | 65 +++++++++++++------ packages/server/src/sdk/app/rows/rows.ts | 13 ++-- .../src/sdk/app/rows/search/external.ts | 13 ++-- .../server/src/sdk/app/rows/search/utils.ts | 1 - packages/server/src/sdk/app/rows/utils.ts | 8 +-- .../src/utilities/rowProcessor/index.ts | 13 ++-- 9 files changed, 89 insertions(+), 70 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index fb992059f4..2d4012dfcf 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -42,6 +42,7 @@ export async function handleRequest( export async function patch(ctx: UserCtx) { const source = await utils.getSource(ctx) + const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body const { row: dataToUpdate } = await inputProcessing( @@ -58,11 +59,11 @@ export async function patch(ctx: UserCtx) { throw { validation: validateResult.errors } } - const beforeRow = await sdk.rows.external.getRow(tableId, _id, { + const beforeRow = await sdk.rows.external.getRow(table._id!, _id, { relationships: true, }) - const response = await handleRequest(Operation.UPDATE, tableId, { + const response = await handleRequest(Operation.UPDATE, source, { id: breakRowIdField(_id), row: dataToUpdate, }) @@ -70,7 +71,7 @@ export async function patch(ctx: UserCtx) { // The id might have been changed, so the refetching would fail. Recalculating the id just in case const updatedId = generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id - const row = await sdk.rows.external.getRow(tableId, updatedId, { + const row = await sdk.rows.external.getRow(table._id!, updatedId, { relationships: true, }) @@ -78,7 +79,6 @@ export async function patch(ctx: UserCtx) { outputProcessing(table, row, { squash: true, preserveLinks: true, - fromViewId: viewId, }), outputProcessing(table, beforeRow, { squash: true, @@ -95,9 +95,9 @@ export async function patch(ctx: UserCtx) { } export async function destroy(ctx: UserCtx) { - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) const _id = ctx.request.body._id - const { row } = await handleRequest(Operation.DELETE, tableId, { + const { row } = await handleRequest(Operation.DELETE, source, { id: breakRowIdField(_id), includeSqlRelationships: IncludeRelationship.EXCLUDE, }) @@ -106,11 +106,11 @@ export async function destroy(ctx: UserCtx) { export async function bulkDestroy(ctx: UserCtx) { const { rows } = ctx.request.body - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) let promises: Promise<{ row: Row; table: Table }>[] = [] for (let row of rows) { promises.push( - handleRequest(Operation.DELETE, tableId, { + handleRequest(Operation.DELETE, source, { id: breakRowIdField(row._id), includeSqlRelationships: IncludeRelationship.EXCLUDE, }) @@ -125,6 +125,7 @@ export async function bulkDestroy(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId + const source = await utils.getSource(ctx) const { tableId } = utils.getSourceId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) const datasource: Datasource = await sdk.datasources.get(datasourceId) @@ -132,7 +133,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { ctx.throw(400, "Datasource has not been configured for plus API.") } const tables = datasource.entities - const response = await handleRequest(Operation.READ, tableId, { + const response = await handleRequest(Operation.READ, source, { id, datasource, includeSqlRelationships: IncludeRelationship.INCLUDE, @@ -156,7 +157,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTable, { tables, filters: { oneOf: { diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 2e5785157d..ac2c09e7a6 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -221,7 +221,7 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - tableId, + sourceId: tableId, } ctx.status = 200 @@ -229,14 +229,15 @@ export async function search(ctx: Ctx) { } export async function validate(ctx: Ctx) { - const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) + const table = await utils.getTableFromSource(source) // external tables are hard to validate currently - if (isExternalTableID(tableId)) { + if (isExternalTableID(table._id!)) { ctx.body = { valid: true, errors: {} } } else { ctx.body = await sdk.rows.utils.validate({ row: ctx.request.body, - tableId, + source, }) } } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 91c0fc966f..e5397ed4a5 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -106,19 +106,11 @@ export async function getSource(ctx: Ctx): Promise
{ return sdk.tables.getTable(tableId) } -export async function validate( - opts: { row: Row } & ({ tableId: string } | { table: Table }) -) { - let fetchedTable: Table - if ("tableId" in opts) { - fetchedTable = await sdk.tables.getTable(opts.tableId) - } else { - fetchedTable = opts.table +export async function getTableFromSource(source: Table | ViewV2) { + if (sdk.views.isView(source)) { + return await sdk.views.getTable(source.id) } - return sdk.rows.utils.validate({ - ...opts, - table: fetchedTable, - }) + return source } function fixBooleanFields({ row, table }: { row: Row; table: Table }) { diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 7630e5638b..24ec2302e7 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -1,5 +1,11 @@ -import { IncludeRelationship, Operation, Row } from "@budibase/types" -import { HTTPError } from "@budibase/backend-core" +import { + IncludeRelationship, + Operation, + Row, + Table, + ViewV2, +} from "@budibase/types" +import { docIds, HTTPError } from "@budibase/backend-core" import { handleRequest } from "../../../api/controllers/row/external" import { breakRowIdField } from "../../../integrations/utils" import sdk from "../../../sdk" @@ -12,11 +18,21 @@ import isEqual from "lodash/fp/isEqual" import { tryExtractingTableAndViewId } from "./utils" export async function getRow( - tableId: string, + sourceId: string | Table | ViewV2, rowId: string, opts?: { relationships?: boolean } ) { - const response = await handleRequest(Operation.READ, tableId, { + let source: Table | ViewV2 + if (typeof sourceId === "string") { + if (docIds.isViewId(sourceId)) { + source = await sdk.views.get(sourceId) + } else { + source = await sdk.tables.getTable(sourceId) + } + } else { + source = sourceId + } + const response = await handleRequest(Operation.READ, source, { id: breakRowIdField(rowId), includeSqlRelationships: opts?.relationships ? IncludeRelationship.INCLUDE @@ -27,45 +43,50 @@ export async function getRow( } export async function save( - tableOrViewId: string, + sourceId: string, inputs: Row, userId: string | undefined ) { - const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - const table = await sdk.tables.getTable(tableId) + const { tableId, viewId } = tryExtractingTableAndViewId(sourceId) + let source: Table | ViewV2 + if (viewId) { + source = await sdk.views.get(viewId) + } else { + source = await sdk.tables.getTable(tableId) + } + const { table: updatedTable, row } = await inputProcessing( userId, - cloneDeep(table), + cloneDeep(source), inputs ) const validateResult = await sdk.rows.utils.validate({ row, - tableId, + source, }) if (!validateResult.valid) { throw { validation: validateResult.errors } } - const response = await handleRequest(Operation.CREATE, tableId, { + const response = await handleRequest(Operation.CREATE, source, { row, }) - if (!isEqual(table, updatedTable)) { + if (sdk.tables.isTable(source) && !isEqual(source, updatedTable)) { await sdk.tables.saveTable(updatedTable) } const rowId = response.row._id if (rowId) { - const row = await getRow(tableId, rowId, { + const row = await getRow(source, rowId, { relationships: true, }) return { ...response, - row: await outputProcessing(table, row, { + row: await outputProcessing(source, row, { preserveLinks: true, squash: true, - fromViewId: viewId, }), } } else { @@ -76,7 +97,14 @@ export async function save( export async function find(tableOrViewId: string, rowId: string): Promise { const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - const row = await getRow(tableId, rowId, { + let source: Table | ViewV2 + if (viewId) { + source = await sdk.views.get(viewId) + } else { + source = await sdk.tables.getTable(tableId) + } + + const row = await getRow(source, rowId, { relationships: true, }) @@ -84,11 +112,10 @@ export async function find(tableOrViewId: string, rowId: string): Promise { throw new HTTPError("Row not found", 404) } - const table = await sdk.tables.getTable(tableId) - // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case - return await outputProcessing(table, row, { + // Preserving links, as the outputProcessing does not support external rows + // yet and we don't need it in this use case + return await outputProcessing(source, row, { squash: true, preserveLinks: true, - fromViewId: viewId, }) } diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts index ef25d06baf..25bdf1fd4f 100644 --- a/packages/server/src/sdk/app/rows/rows.ts +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -1,9 +1,6 @@ import { db as dbCore, context, docIds } from "@budibase/backend-core" import { Database, Row } from "@budibase/types" -import { - extractViewInfoFromID, - getRowParams, -} from "../../../db/utils" +import { extractViewInfoFromID, getRowParams } from "../../../db/utils" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" @@ -36,13 +33,13 @@ function pickApi(tableOrViewId: string) { } export async function save( - tableOrViewId: string, + sourceId: string, row: Row, userId: string | undefined ) { - return pickApi(tableOrViewId).save(tableOrViewId, row, userId) + return pickApi(sourceId).save(sourceId, row, userId) } -export async function find(tableOrViewId: string, rowId: string) { - return pickApi(tableOrViewId).find(tableOrViewId, rowId) +export async function find(sourceId: string, rowId: string) { + return pickApi(sourceId).find(sourceId, rowId) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 5584b3f110..a41ae8dcda 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -106,9 +106,9 @@ export async function search( includeSqlRelationships: IncludeRelationship.INCLUDE, } const [{ rows, rawResponseSize }, totalRows] = await Promise.all([ - handleRequest(Operation.READ, tableId, parameters), + handleRequest(Operation.READ, source, parameters), countRows - ? handleRequest(Operation.COUNT, tableId, parameters) + ? handleRequest(Operation.COUNT, source, parameters) : Promise.resolve(undefined), ]) @@ -200,7 +200,7 @@ export async function exportRows( } let result = await search( - { tableId, query: requestQuery, sort, sortOrder }, + { sourceId: table._id!, query: requestQuery, sort, sortOrder }, table ) let rows: Row[] = [] @@ -256,10 +256,10 @@ export async function exportRows( } export async function fetch(tableId: string): Promise { - const response = await handleRequest(Operation.READ, tableId, { + const table = await sdk.tables.getTable(tableId) + const response = await handleRequest(Operation.READ, table, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) - const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response.rows, { preserveLinks: true, squash: true, @@ -267,7 +267,8 @@ export async function fetch(tableId: string): Promise { } export async function fetchRaw(tableId: string): Promise { - const response = await handleRequest(Operation.READ, tableId, { + const table = await sdk.tables.getTable(tableId) + const response = await handleRequest(Operation.READ, table, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) return response.rows diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index d727e58887..6548f963b8 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -9,7 +9,6 @@ import { SearchResponse, Row, RowSearchParams, - ViewV2, } from "@budibase/types" import { db as dbCore, context } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 3899009f13..f17d3e4a03 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -21,6 +21,7 @@ import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds } from "@budibase/backend-core" +import { getTableFromSource } from "../../../api/controllers/row/utils" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -149,12 +150,7 @@ export async function validate({ valid: boolean errors: Record }> { - let table: Table - if (sdk.views.isView(source)) { - table = await sdk.views.getTable(source.id) - } else { - table = source - } + const table = await getTableFromSource(source) const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 73768fdd57..9aa53e18c1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -19,6 +19,7 @@ import { RowAttachment, Table, User, + ViewV2, } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { @@ -34,7 +35,11 @@ import { PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" import { processString } from "@budibase/string-templates" -import { isUserMetadataTable } from "../../api/controllers/row/utils" +import { + getTableFromSource, + isUserMetadataTable, +} from "../../api/controllers/row/utils" +import sdk from "../../sdk" export * from "./utils" export * from "./attachments" @@ -170,11 +175,12 @@ export function coerce(row: any, type: string) { */ export async function inputProcessing( userId: string | null | undefined, - table: Table, + source: Table | ViewV2, row: Row, opts?: AutoColumnProcessingOpts ) { const clonedRow = cloneDeep(row) + const table = await getTableFromSource(source) const dontCleanseKeys = ["type", "_id", "_rev", "tableId"] for (const [key, value] of Object.entries(clonedRow)) { @@ -243,14 +249,13 @@ export async function inputProcessing( * @returns the enriched rows will be returned. */ export async function outputProcessing( - table: Table, + source: Table | ViewV2, rows: T, opts: { squash?: boolean preserveLinks?: boolean fromRow?: Row skipBBReferences?: boolean - fromViewId?: string aggregations?: Aggregation[] } = { squash: true, From 0eb90cfbea9e8dc16ecc917c1994382fdb901000 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 16:35:53 +0100 Subject: [PATCH 006/105] Type checks pass, now to find out how much stuff I've broken. --- packages/backend-core/src/db/lucene.ts | 16 +++---- .../backend-core/src/db/tests/lucene.spec.ts | 4 +- .../src/api/controllers/row/external.ts | 2 +- .../src/api/controllers/row/internal.ts | 31 +++++------- .../src/api/controllers/row/staticFormula.ts | 32 +++++-------- .../src/api/controllers/row/utils/utils.ts | 18 +------ .../src/api/controllers/table/external.ts | 5 +- .../server/src/api/controllers/table/utils.ts | 3 +- .../src/api/routes/tests/search.spec.ts | 12 ++--- .../src/api/routes/tests/templates.spec.ts | 2 +- packages/server/src/db/linkedRows/index.ts | 26 +++++----- .../integrations/tests/googlesheets.spec.ts | 22 ++++----- packages/server/src/sdk/app/rows/external.ts | 11 +---- packages/server/src/sdk/app/rows/internal.ts | 47 ++++++++++--------- .../sdk/app/rows/search/internal/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 20 +++++--- .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++- .../sdk/app/rows/search/tests/search.spec.ts | 10 ++-- .../sdk/app/rows/search/tests/utils.spec.ts | 12 ++--- .../src/sdk/app/rows/tests/utils.spec.ts | 42 ++++++++--------- packages/server/src/sdk/app/rows/utils.ts | 7 +++ packages/server/src/sdk/app/views/index.ts | 3 +- .../src/utilities/rowProcessor/index.ts | 23 +++++++-- .../tests/inputProcessing.spec.ts | 10 ++-- 24 files changed, 183 insertions(+), 188 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f5ad7e6433..b17c3ddf0d 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -79,8 +79,8 @@ export class QueryBuilder { return this } - setTable(tableId: string) { - this.#query.equal!.tableId = tableId + setSource(sourceId: string) { + this.#query.equal!.tableId = sourceId return this } @@ -638,8 +638,8 @@ async function recursiveSearch( .setSortOrder(params.sortOrder) .setSortType(params.sortType) - if (params.tableId) { - queryBuilder.setTable(params.tableId) + if (params.sourceId) { + queryBuilder.setSource(params.sourceId) } const page = await queryBuilder.run() @@ -672,8 +672,8 @@ export async function paginatedSearch( if (params.version) { search.setVersion(params.version) } - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } if (params.sort) { search @@ -695,8 +695,8 @@ export async function paginatedSearch( // Try fetching 1 row in the next page to see if another page of results // exists or not search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.tableId) { - search.setTable(params.tableId) + if (params.sourceId) { + search.setSource(params.sourceId) } const nextResults = await search.run() diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index c41bdf88d1..8747f56a4b 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -366,7 +366,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, limit: 1, sort: "property", sortType: SortType.STRING, @@ -390,7 +390,7 @@ describe("lucene", () => { }, }, { - tableId: TABLE_ID, + sourceId: TABLE_ID, query: {}, } ) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 2d4012dfcf..11b6559896 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -45,7 +45,7 @@ export async function patch(ctx: UserCtx) { const table = await utils.getTableFromSource(source) const { _id, ...rowData } = ctx.request.body - const { row: dataToUpdate } = await inputProcessing( + const dataToUpdate = await inputProcessing( ctx.user?._id, cloneDeep(source), rowData diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 33e3c7707b..f5cb42f81d 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -21,18 +21,19 @@ import { import sdk from "../../../sdk" import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils" import { flatten } from "lodash" +import { findRow } from "../../../sdk/app/rows/internal" export async function patch(ctx: UserCtx) { - const { tableId, viewId } = utils.getSourceId(ctx) + const { tableId } = utils.getSourceId(ctx) + const source = await utils.getSource(ctx) + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source const inputs = ctx.request.body const isUserTable = tableId === InternalTables.USER_METADATA let oldRow - const dbTable = await sdk.tables.getTable(tableId) try { - oldRow = await outputProcessing( - dbTable, - await utils.findRow(tableId, inputs._id!) - ) + oldRow = await outputProcessing(source, await findRow(tableId, inputs._id!)) } catch (err) { if (isUserTable) { // don't include the rev, it'll be the global rev @@ -48,22 +49,18 @@ export async function patch(ctx: UserCtx) { // need to build up full patch fields before coerce let combinedRow: any = cloneDeep(oldRow) for (let key of Object.keys(inputs)) { - if (!dbTable.schema[key]) continue + if (!table.schema[key]) continue combinedRow[key] = inputs[key] } // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const tableClone = cloneDeep(table) // this returns the table and row incase they have been updated - let { table, row } = await inputProcessing( - ctx.user?._id, - tableClone, - combinedRow - ) + let row = await inputProcessing(ctx.user?._id, tableClone, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -87,10 +84,8 @@ export async function patch(ctx: UserCtx) { return { row: ctx.body as Row, table, oldRow } } - const result = await finaliseRow(table, row, { - oldTable: dbTable, + const result = await finaliseRow(source, row, { updateFormula: true, - fromViewId: viewId, }) return { ...result, oldRow } @@ -186,7 +181,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { sdk.tables.getTable(tableId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - let row = await utils.findRow(tableId, rowId) + let row = await findRow(tableId, rowId) row = await outputProcessing(table, row) const linkVals = links as LinkDocumentValue[] diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 777379db14..386dee7b4a 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -4,10 +4,11 @@ import { processFormulas, } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, FormulaType, FieldType } from "@budibase/types" +import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types" import * as linkRows from "../../../db/linkedRows" import isEqual from "lodash/isEqual" import { cloneDeep } from "lodash/fp" +import sdk from "../../../sdk" /** * This function runs through a list of enriched rows, looks at the rows which @@ -121,33 +122,26 @@ export async function updateAllFormulasInTable(table: Table) { * expects the row to be totally enriched/contain all relationships. */ export async function finaliseRow( - table: Table, + source: Table | ViewV2, row: Row, - { - oldTable, - updateFormula, - fromViewId, - }: { oldTable?: Table; updateFormula: boolean; fromViewId?: string } = { - updateFormula: true, - } + opts?: { updateFormula: boolean } ) { const db = context.getAppDB() + const { updateFormula = true } = opts || {} + const table = sdk.views.isView(source) + ? await sdk.views.getTable(source.id) + : source + row.type = "row" // process the row before return, to include relationships - let enrichedRow = (await outputProcessing(table, cloneDeep(row), { + let enrichedRow = await outputProcessing(source, cloneDeep(row), { squash: false, - })) as Row + }) // use enriched row to generate formulas for saving, specifically only use as context row = await processFormulas(table, row, { dynamic: false, contextRows: [enrichedRow], }) - // don't worry about rev, tables handle rev/lastID updates - // if another row has been written since processing this will - // handle the auto ID clash - if (oldTable && !isEqual(oldTable, table)) { - await db.put(table) - } const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev @@ -158,8 +152,6 @@ export async function finaliseRow( if (updateFormula) { await updateRelatedFormula(table, enrichedRow) } - const squashed = await linkRows.squashLinks(table, enrichedRow, { - fromViewId, - }) + const squashed = await linkRows.squashLinks(source, enrichedRow) return { row: enrichedRow, squashed, table } } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index e5397ed4a5..0f565b6951 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -1,6 +1,6 @@ import * as utils from "../../../../db/utils" -import { context, docIds } from "@budibase/backend-core" +import { docIds } from "@budibase/backend-core" import { Aggregation, Ctx, @@ -20,7 +20,6 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" -import { getFullUser } from "../../../../utilities/users" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -60,21 +59,6 @@ export async function processRelationshipFields( return row } -export async function findRow(tableId: string, rowId: string) { - const db = context.getAppDB() - let row: Row - // TODO remove special user case in future - if (tableId === utils.InternalTables.USER_METADATA) { - row = await getFullUser(rowId) - } else { - row = await db.get(rowId) - } - if (row.tableId !== tableId) { - throw "Supplied tableId does not match the rows tableId" - } - return row -} - export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } { // top priority, use the URL first if (ctx.params?.sourceId) { diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index c3356919c8..5b15d3d9c7 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -113,11 +113,10 @@ export async function bulkImport( const processed = await inputProcessing(ctx.user?._id, table, row, { noAutoRelationships: true, }) - parsedRows.push(processed.row) - table = processed.table + parsedRows.push(processed) } - await handleRequest(Operation.BULK_UPSERT, table._id!, { + await handleRequest(Operation.BULK_UPSERT, table, { rows: parsedRows, }) await events.rows.imported(table, parsedRows.length) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 269f079ae8..d568e5f33e 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -139,8 +139,7 @@ export async function importToRows( const processed = await inputProcessing(user?._id, table, row, { noAutoRelationships: true, }) - row = processed.row - table = processed.table + row = processed // However here we must reference the original table, as we want to mutate // the real schema of the table passed in, not the clone used for diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c770c4e460..5788d9195a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -157,7 +157,7 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), this.query) } else { - return config.api.row.search(this.query.tableId, this.query) + return config.api.row.search(this.query.sourceId, this.query) } } @@ -327,8 +327,8 @@ describe.each([ } } - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: table._id! }) + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, sourceId: table._id! }) } function expectQuery(query: SearchFilters) { @@ -1898,7 +1898,7 @@ describe.each([ let { rows: fullRowList } = await config.api.row.search( table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, } ) @@ -1909,7 +1909,7 @@ describe.each([ rowCount: number = 0 do { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 1, paginate: true, query: {}, @@ -1933,7 +1933,7 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, limit: 3, query: {}, bookmark, diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 6f4d468a68..4290b4386f 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -113,7 +113,7 @@ describe("/templates", () => { expect(users.name).toBe("Users") const { rows } = await config.api.row.search(agencyProjects._id!, { - tableId: agencyProjects._id!, + sourceId: agencyProjects._id!, query: {}, }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 4222f9b5e4..6e65ab36d1 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -255,31 +255,31 @@ export type SquashTableFields = Record * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( - table: Table, - enriched: T, - options?: { - fromViewId?: string - } + source: Table | ViewV2, + enriched: T ): Promise { const allowRelationshipSchemas = await features.flags.isEnabled( FeatureFlag.ENRICHED_RELATIONSHIPS ) let viewSchema: Record = {} - if (options?.fromViewId) { - const view = Object.values(table.views || {}).find( - (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId - ) - - if (view && helpers.views.isCalculationView(view)) { + if (sdk.views.isView(source)) { + if (helpers.views.isCalculationView(source)) { return enriched } - if (allowRelationshipSchemas && view) { - viewSchema = view.schema || {} + if (allowRelationshipSchemas) { + viewSchema = source.schema || {} } } + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 34be1c0c6c..91addf8a50 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -219,7 +219,7 @@ describe("Google Sheets Integration", () => { }) let resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -228,7 +228,7 @@ describe("Google Sheets Integration", () => { while (resp.hasNextPage) { resp = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: {}, paginate: true, limit: 10, @@ -637,7 +637,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Foo", @@ -651,7 +651,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not equals filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEqual: { name: "Foo", @@ -666,7 +666,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { empty: { name: null, @@ -679,7 +679,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not empty filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { notEmpty: { name: null, @@ -692,7 +692,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with one of filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { name: ["Foo", "Bar"], @@ -707,7 +707,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with fuzzy filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { fuzzy: { name: "oo", @@ -721,7 +721,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with range filter", async () => { const response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { range: { name: { @@ -750,7 +750,7 @@ describe("Google Sheets Integration", () => { }) let response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, @@ -759,7 +759,7 @@ describe("Google Sheets Integration", () => { while (response.hasNextPage) { response = await config.api.row.search(table._id!, { - tableId: table._id!, + sourceId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts index 24ec2302e7..060ef3738a 100644 --- a/packages/server/src/sdk/app/rows/external.ts +++ b/packages/server/src/sdk/app/rows/external.ts @@ -14,7 +14,6 @@ import { outputProcessing, } from "../../../utilities/rowProcessor" import cloneDeep from "lodash/fp/cloneDeep" -import isEqual from "lodash/fp/isEqual" import { tryExtractingTableAndViewId } from "./utils" export async function getRow( @@ -55,11 +54,7 @@ export async function save( source = await sdk.tables.getTable(tableId) } - const { table: updatedTable, row } = await inputProcessing( - userId, - cloneDeep(source), - inputs - ) + const row = await inputProcessing(userId, cloneDeep(source), inputs) const validateResult = await sdk.rows.utils.validate({ row, @@ -73,10 +68,6 @@ export async function save( row, }) - if (sdk.tables.isTable(source) && !isEqual(source, updatedTable)) { - await sdk.tables.saveTable(updatedTable) - } - const rowId = response.row._id if (rowId) { const row = await getRow(source, rowId, { diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index b5b3437e5e..f51bcc37a4 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -1,5 +1,5 @@ import { context, db } from "@budibase/backend-core" -import { Row } from "@budibase/types" +import { Row, Table, ViewV2 } from "@budibase/types" import sdk from "../../../sdk" import cloneDeep from "lodash/fp/cloneDeep" import { finaliseRow } from "../../../api/controllers/row/staticFormula" @@ -10,7 +10,7 @@ import { import * as linkRows from "../../../db/linkedRows" import { InternalTables } from "../../../db/utils" import { getFullUser } from "../../../utilities/users" -import { tryExtractingTableAndViewId } from "./utils" +import { getSource, tryExtractingTableAndViewId } from "./utils" export async function save( tableOrViewId: string, @@ -20,21 +20,28 @@ export async function save( const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) inputs.tableId = tableId + let source: Table | ViewV2 + let table: Table + if (viewId) { + source = await sdk.views.get(viewId) + table = await sdk.views.getTable(viewId) + } else { + source = await sdk.tables.getTable(tableId) + table = source + } + if (!inputs._rev && !inputs._id) { inputs._id = db.generateRowID(inputs.tableId) } - // this returns the table and row incase they have been updated - const dbTable = await sdk.tables.getTable(inputs.tableId) - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(dbTable) + const sourceClone = cloneDeep(source) - let { table, row } = await inputProcessing(userId, tableClone, inputs) + let row = await inputProcessing(userId, sourceClone, inputs) const validateResult = await sdk.rows.utils.validate({ row, - table, + source, }) if (!validateResult.valid) { @@ -49,24 +56,18 @@ export async function save( table, })) as Row - return finaliseRow(table, row, { - oldTable: dbTable, - updateFormula: true, - fromViewId: viewId, + return finaliseRow(table, row, { updateFormula: true }) +} + +export async function find(sourceId: string, rowId: string): Promise { + const source = await getSource(sourceId) + return await outputProcessing(source, await findRow(sourceId, rowId), { + squash: true, }) } -export async function find(tableOrViewId: string, rowId: string): Promise { - const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId) - - const table = await sdk.tables.getTable(tableId) - let row = await findRow(tableId, rowId) - - row = await outputProcessing(table, row, { squash: true, fromViewId: viewId }) - return row -} - -async function findRow(tableId: string, rowId: string) { +export async function findRow(sourceId: string, rowId: string) { + const { tableId } = tryExtractingTableAndViewId(sourceId) const db = context.getAppDB() let row: Row // TODO remove special user case in future diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index 6617fc376c..c9e2aba237 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -64,7 +64,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { let searchResponse = await sdk.rows.search({ - tableId, + sourceId: tableId, query, sort, sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 2c149e5b21..24a7de1307 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -8,21 +8,30 @@ import { SortType, Table, User, + ViewV2, } from "@budibase/types" import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" +import sdk from "../../../../" export async function search( options: RowSearchParams, - table: Table + source: Table | ViewV2 ): Promise> { - const { tableId } = options + const { sourceId } = options + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } const { paginate, query } = options const params: RowSearchParams = { - tableId: options.tableId, + sourceId: options.sourceId, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, @@ -50,7 +59,7 @@ export async function search( // Enrich search results with relationships if (response.rows && response.rows.length) { // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { + if (sourceId === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } @@ -59,9 +68,8 @@ export async function search( response.rows = response.rows.map((r: any) => pick(r, fields)) } - response.rows = await outputProcessing(table, response.rows, { + response.rows = await outputProcessing(source, response.rows, { squash: true, - fromViewId: options.viewId, }) } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index b5bf8e752f..90fb082214 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -15,6 +15,7 @@ import { SortType, SqlClient, Table, + ViewV2, } from "@budibase/types" import { buildInternalRelationships, @@ -292,11 +293,18 @@ function resyncDefinitionsRequired(status: number, message: string) { export async function search( options: RowSearchParams, - table: Table, + source: Table | ViewV2, opts?: { retrying?: boolean } ): Promise> { let { paginate, query, ...params } = cloneDeep(options) + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) // make sure we have the mapped/latest table @@ -406,7 +414,6 @@ export async function search( let finalRows = await outputProcessing(table, processed, { preserveLinks: true, squash: true, - fromViewId: options.viewId, aggregations: options.aggregations, }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index e7fd095865..194f2dd4e3 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -122,7 +122,7 @@ describe.each([ it("querying by fields will always return data attribute columns", async () => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -142,7 +142,7 @@ describe.each([ it("will decode _id in oneOf query", async () => { await config.doInContext(config.appId, async () => { const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: { oneOf: { _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], @@ -174,7 +174,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, }) expect(result.rows).toHaveLength(10) @@ -205,7 +205,7 @@ describe.each([ }, }) const result = await search({ - tableId: table._id!, + sourceId: table._id!, query: {}, fields: ["name", "age"], }) @@ -229,7 +229,7 @@ describe.each([ async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - tableId: table._id!, + sourceId: table._id!, query: { $or: { conditions: [ diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index 0698f727df..e3f241f15a 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - tableId, + sourceId: tableId, query: { equal: { "1:user": email, @@ -90,10 +90,8 @@ describe.each([tableWithUserCol, tableWithUsersCol])( }) it("shouldn't error if no query supplied", () => { - const params: any = { - tableId, - } - const output = searchInputMapping(col, params) + // @ts-expect-error - intentionally passing in a bad type + const output = searchInputMapping(col, { sourceId: tableId }) expect(output.query).toBeUndefined() }) } diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 55cdf9ea20..548b2b6bc9 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -33,7 +33,7 @@ describe("validate", () => { it("should accept empty values", async () => { const row = {} const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) expect(output.errors).toEqual({}) }) @@ -43,7 +43,7 @@ describe("validate", () => { time: `${hour()}:${minute()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -52,7 +52,7 @@ describe("validate", () => { time: `${hour()}:${minute()}:${second()}`, } const table = getTable() - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -67,7 +67,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ['"time" is not a valid time'] }) }) @@ -91,7 +91,7 @@ describe("validate", () => { `${generator.integer({ min: 11, max: 23 })}:${minute()}`, ])("should accept values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -100,7 +100,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -125,7 +125,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 12 })}:${minute()}`, ])("should accept values before config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) }) @@ -134,7 +134,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after config value (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:16:17"], @@ -156,7 +156,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -166,7 +166,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -178,7 +178,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -199,7 +199,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -208,7 +208,7 @@ describe("validate", () => { "should reject values out range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 10:00"], @@ -226,7 +226,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -237,7 +237,7 @@ describe("validate", () => { table.schema.time.constraints = { presence: true, } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["can't be blank"] }) }) @@ -257,7 +257,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -267,7 +267,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -279,7 +279,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], @@ -301,7 +301,7 @@ describe("validate", () => { "should accept values in range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(true) } ) @@ -311,7 +311,7 @@ describe("validate", () => { `${generator.integer({ min: 0, max: 9 })}:${minute()}`, ])("should reject values before range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no earlier than 10:00"], @@ -323,7 +323,7 @@ describe("validate", () => { `${generator.integer({ min: 16, max: 23 })}:${minute()}`, ])("should reject values after range (%s)", async time => { const row = { time } - const output = await validate({ table, tableId: table._id!, row }) + const output = await validate({ source: table, row }) expect(output.valid).toBe(false) expect(output.errors).toEqual({ time: ["must be no later than 15:00"], diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index f17d3e4a03..d5c0560d9b 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -321,3 +321,10 @@ export function tryExtractingTableAndViewId(tableOrViewId: string) { return { tableId: tableOrViewId } } + +export function getSource(tableOrViewId: string) { + if (docIds.isViewId(tableOrViewId)) { + return sdk.views.get(tableOrViewId) + } + return sdk.tables.getTable(tableOrViewId) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index f911385dc1..4251383712 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -40,7 +40,8 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } -export async function getTable(viewId: string): Promise
{ +export async function getTable(view: string | ViewV2): Promise
{ + const viewId = typeof view === "string" ? view : view.id const cached = context.getTableForView(viewId) if (cached) { return cached diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 9aa53e18c1..24c8d11bd1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -73,6 +73,7 @@ export async function processAutoColumn( // check its not user table, or whether any of the processing options have been disabled const shouldUpdateUserFields = !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser + let tableMutated = false for (let [key, schema] of Object.entries(table.schema)) { if (!schema.autocolumn) { continue @@ -105,10 +106,17 @@ export async function processAutoColumn( row[key] = schema.lastID + 1 schema.lastID++ table.schema[key] = schema + tableMutated = true } break } } + + if (tableMutated) { + const db = context.getAppDB() + const resp = await db.put(table) + table._rev = resp.rev + } } async function processDefaultValues(table: Table, row: Row) { @@ -235,8 +243,7 @@ export async function inputProcessing( await processAutoColumn(userId, table, clonedRow, opts) await processDefaultValues(table, clonedRow) - - return { table, row: clonedRow } + return clonedRow } /** @@ -271,6 +278,14 @@ export async function outputProcessing( } else { safeRows = rows } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + // attach any linked row information let enriched = !opts.preserveLinks ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { @@ -360,9 +375,7 @@ export async function outputProcessing( enriched = await processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { - enriched = await linkRows.squashLinks(table, enriched, { - fromViewId: opts?.fromViewId, - }) + enriched = await linkRows.squashLinks(source, enriched) } // remove null properties to match internal API diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 244ea3794c..1a75cd6830 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -65,7 +65,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferenceMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledTimes( 1 @@ -117,7 +117,7 @@ describe("rowProcessor - inputProcessing", () => { processInputBBReferencesMock.mockResolvedValue(user) - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).toHaveBeenCalledTimes( 1 @@ -164,7 +164,7 @@ describe("rowProcessor - inputProcessing", () => { name: "Jack", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({ ...newRow, user: undefined }) @@ -207,7 +207,7 @@ describe("rowProcessor - inputProcessing", () => { user: userValue, } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) if (userValue === undefined) { // The 'user' field is omitted @@ -262,7 +262,7 @@ describe("rowProcessor - inputProcessing", () => { user: "123", } - const { row } = await inputProcessing(userId, table, newRow) + const row = await inputProcessing(userId, table, newRow) expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled() expect(row).toEqual({ From e3256cb005ec6f8517e5db995f8c555ba00f7b38 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 17:46:38 +0100 Subject: [PATCH 007/105] Fix row.spec.ts. --- packages/backend-core/src/db/lucene.ts | 1 - .../src/api/controllers/row/external.ts | 4 ++-- .../server/src/api/controllers/row/index.ts | 4 ++-- .../src/api/controllers/row/utils/utils.ts | 8 +++++++- .../server/src/api/controllers/table/index.ts | 9 ++------- .../server/src/api/routes/tests/row.spec.ts | 12 ++++++++--- packages/server/src/db/linkedRows/index.ts | 7 ++++--- packages/server/src/sdk/app/rows/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 2 +- .../src/sdk/app/rows/search/internal/sqs.ts | 20 ++++++++----------- 10 files changed, 36 insertions(+), 33 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index b17c3ddf0d..7f58c7068e 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -612,7 +612,6 @@ async function runQuery( * limit {number} The number of results to fetch * bookmark {string|null} Current bookmark in the recursive search * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} */ async function recursiveSearch( dbName: string, diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 11b6559896..18a9be5087 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,11 +76,11 @@ export async function patch(ctx: UserCtx) { }) const [enrichedRow, oldRow] = await Promise.all([ - outputProcessing(table, row, { + outputProcessing(source, row, { squash: true, preserveLinks: true, }), - outputProcessing(table, beforeRow, { + outputProcessing(source, beforeRow, { squash: true, preserveLinks: true, }), diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index ac2c09e7a6..84b5edcc12 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -207,7 +207,7 @@ export async function destroy(ctx: UserCtx) { } export async function search(ctx: Ctx) { - const { tableId } = utils.getSourceId(ctx) + const { tableId, viewId } = utils.getSourceId(ctx) await context.ensureSnippetContext(true) @@ -221,7 +221,7 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - sourceId: tableId, + sourceId: viewId || tableId, } ctx.status = 200 diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 0f565b6951..45abd93930 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -112,7 +112,7 @@ function fixBooleanFields({ row, table }: { row: Row; table: Table }) { export async function sqlOutputProcessing( rows: DatasourcePlusQueryResponse, - table: Table, + source: Table | ViewV2, tables: Record, relationships: RelationshipsJson[], opts?: { sqs?: boolean; aggregations?: Aggregation[] } @@ -120,6 +120,12 @@ export async function sqlOutputProcessing( if (isKnexEmptyReadResponse(rows)) { return [] } + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } let finalRows: { [key: string]: Row } = {} for (let row of rows as Row[]) { let rowId = row._id diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 0e16077092..7bec1581b4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -33,7 +33,7 @@ import { import sdk from "../../../sdk" import { jsonFromCsvString } from "../../../utilities/csv" import { builderSocket } from "../../../websockets" -import { cloneDeep, isEqual } from "lodash" +import { cloneDeep } from "lodash" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -149,12 +149,7 @@ export async function bulkImport( ctx: UserCtx ) { const tableId = ctx.params.tableId - let tableBefore = await sdk.tables.getTable(tableId) - let tableAfter = await pickApi({ tableId }).bulkImport(ctx) - - if (!isEqual(tableBefore, tableAfter)) { - await sdk.tables.saveTable(tableAfter) - } + await pickApi({ tableId }).bulkImport(ctx) // right now we don't trigger anything for bulk import because it // can only be done in the builder, but in the future we may need to diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index dc03a21d6d..467faa8e06 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,7 +76,7 @@ async function waitForEvent( } describe.each([ - ["internal", undefined], + ["lucene", undefined], ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], @@ -2453,9 +2453,15 @@ describe.each([ let flagCleanup: (() => void) | undefined beforeAll(async () => { - flagCleanup = setCoreEnv({ + const env = { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`, - }) + } + if (isSqs) { + env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS` + } else { + env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS` + } + flagCleanup = setCoreEnv(env) const aux2Table = await config.api.table.save(saveTableRequest()) const aux2Data = await config.api.row.save(aux2Table._id!, {}) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 6e65ab36d1..bc37fbef42 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -248,10 +248,11 @@ function getPrimaryDisplayValue(row: Row, table?: Table) { export type SquashTableFields = Record /** - * This function will take the given enriched rows and squash the links to only contain the primary display field. - * @param table The table from which the rows originated. + * This function will take the given enriched rows and squash the links to only + * contain the primary display field. + * + * @param source The table or view from which the rows originated. * @param enriched The pre-enriched rows (full docs) which are to be squashed. - * @param squashFields Per link column (key) define which columns are allowed while squashing. * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index f51bcc37a4..13054f43f4 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -56,7 +56,7 @@ export async function save( table, })) as Row - return finaliseRow(table, row, { updateFormula: true }) + return finaliseRow(source, row, { updateFormula: true }) } export async function find(sourceId: string, rowId: string): Promise { diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 24a7de1307..65cf4053d7 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -31,7 +31,7 @@ export async function search( const { paginate, query } = options const params: RowSearchParams = { - sourceId: options.sourceId, + sourceId: table._id!, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 90fb082214..5a9e1ddf24 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -308,8 +308,8 @@ export async function search( const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) // make sure we have the mapped/latest table - if (table?._id) { - table = allTablesMap[table?._id] + if (table._id) { + table = allTablesMap[table._id] } if (!table) { throw new Error("Unable to find table") @@ -322,13 +322,6 @@ export async function search( documentType: DocumentType.ROW, } - let fields = options.fields - if (fields === undefined) { - fields = buildInternalFieldList(table, allTables, { relationships }) - } else { - fields = fields.map(f => mapToUserColumn(f)) - } - if (options.aggregations) { options.aggregations = options.aggregations.map(a => { a.field = mapToUserColumn(a.field) @@ -350,7 +343,10 @@ export async function search( tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, - resource: { fields, aggregations: options.aggregations }, + resource: { + fields: buildInternalFieldList(table, allTables, { relationships }), + aggregations: options.aggregations, + }, relationships, } @@ -394,7 +390,7 @@ export async function search( // make sure JSON columns corrected const processed = builder.convertJsonStringColumns( table, - await sqlOutputProcessing(rows, table, allTablesMap, relationships, { + await sqlOutputProcessing(rows, source, allTablesMap, relationships, { sqs: true, aggregations: options.aggregations, }) @@ -411,7 +407,7 @@ export async function search( } // get the rows - let finalRows = await outputProcessing(table, processed, { + let finalRows = await outputProcessing(source, processed, { preserveLinks: true, squash: true, aggregations: options.aggregations, From f475454bce2a5daa3c745716c853be0a371239c1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 18:07:31 +0100 Subject: [PATCH 008/105] Fix search API break. --- packages/backend-core/src/db/lucene.ts | 16 ++++++------ .../backend-core/src/db/tests/lucene.spec.ts | 4 +-- .../server/src/api/controllers/row/index.ts | 3 ++- .../server/src/api/controllers/row/views.ts | 11 +++----- .../src/api/routes/tests/search.spec.ts | 16 +++++++----- .../src/api/routes/tests/templates.spec.ts | 2 +- .../integrations/tests/googlesheets.spec.ts | 22 ++++++++-------- packages/server/src/sdk/app/rows/search.ts | 25 ++++++++----------- .../src/sdk/app/rows/search/external.ts | 2 +- .../sdk/app/rows/search/internal/internal.ts | 2 +- .../sdk/app/rows/search/internal/lucene.ts | 7 +++--- .../sdk/app/rows/search/tests/search.spec.ts | 10 ++++---- .../sdk/app/rows/search/tests/utils.spec.ts | 8 +++--- packages/types/src/sdk/row.ts | 5 ++-- 14 files changed, 66 insertions(+), 67 deletions(-) diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 7f58c7068e..0206bb2140 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -79,8 +79,8 @@ export class QueryBuilder { return this } - setSource(sourceId: string) { - this.#query.equal!.tableId = sourceId + setTable(tableId: string) { + this.#query.equal!.tableId = tableId return this } @@ -637,8 +637,8 @@ async function recursiveSearch( .setSortOrder(params.sortOrder) .setSortType(params.sortType) - if (params.sourceId) { - queryBuilder.setSource(params.sourceId) + if (params.tableId) { + queryBuilder.setTable(params.tableId) } const page = await queryBuilder.run() @@ -671,8 +671,8 @@ export async function paginatedSearch( if (params.version) { search.setVersion(params.version) } - if (params.sourceId) { - search.setSource(params.sourceId) + if (params.tableId) { + search.setTable(params.tableId) } if (params.sort) { search @@ -694,8 +694,8 @@ export async function paginatedSearch( // Try fetching 1 row in the next page to see if another page of results // exists or not search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.sourceId) { - search.setSource(params.sourceId) + if (params.tableId) { + search.setTable(params.tableId) } const nextResults = await search.run() diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts index 8747f56a4b..c41bdf88d1 100644 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ b/packages/backend-core/src/db/tests/lucene.spec.ts @@ -366,7 +366,7 @@ describe("lucene", () => { }, }, { - sourceId: TABLE_ID, + tableId: TABLE_ID, limit: 1, sort: "property", sortType: SortType.STRING, @@ -390,7 +390,7 @@ describe("lucene", () => { }, }, { - sourceId: TABLE_ID, + tableId: TABLE_ID, query: {}, } ) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 84b5edcc12..680a7671d5 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -221,7 +221,8 @@ export async function search(ctx: Ctx) { const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, - sourceId: viewId || tableId, + tableId, + viewId, } ctx.status = 200 diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 7008c5e0be..fa75990136 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,8 +3,6 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, - RequiredKeys, - RowSearchParams, SearchFilterKey, LogicalOperator, Aggregation, @@ -83,9 +81,9 @@ export async function searchView( field, })) - const searchOptions: RequiredKeys & - RequiredKeys> = { - sourceId: view.id, + const result = await sdk.rows.search({ + viewId: view.id, + tableId: view.tableId, query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), @@ -94,9 +92,8 @@ export async function searchView( paginate: body.paginate, countRows: body.countRows, aggregations, - } + }) - const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5788d9195a..090514250d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -157,7 +157,11 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), this.query) } else { - return config.api.row.search(this.query.sourceId, this.query) + const sourceId = this.query.viewId || this.query.tableId + if (!sourceId) { + throw new Error("No source ID provided") + } + return config.api.row.search(sourceId, this.query) } } @@ -327,8 +331,8 @@ describe.each([ } } - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, sourceId: table._id! }) + function expectSearch(query: Omit) { + return new SearchAssertion({ ...query, tableId: table._id! }) } function expectQuery(query: SearchFilters) { @@ -1898,7 +1902,7 @@ describe.each([ let { rows: fullRowList } = await config.api.row.search( table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, } ) @@ -1909,7 +1913,7 @@ describe.each([ rowCount: number = 0 do { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, limit: 1, paginate: true, query: {}, @@ -1933,7 +1937,7 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, limit: 3, query: {}, bookmark, diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index 4290b4386f..6f4d468a68 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -113,7 +113,7 @@ describe("/templates", () => { expect(users.name).toBe("Users") const { rows } = await config.api.row.search(agencyProjects._id!, { - sourceId: agencyProjects._id!, + tableId: agencyProjects._id!, query: {}, }) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 91addf8a50..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -219,7 +219,7 @@ describe("Google Sheets Integration", () => { }) let resp = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, paginate: true, limit: 10, @@ -228,7 +228,7 @@ describe("Google Sheets Integration", () => { while (resp.hasNextPage) { resp = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: {}, paginate: true, limit: 10, @@ -637,7 +637,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with equals filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Foo", @@ -651,7 +651,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not equals filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { notEqual: { name: "Foo", @@ -666,7 +666,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with empty filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { empty: { name: null, @@ -679,7 +679,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with not empty filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { notEmpty: { name: null, @@ -692,7 +692,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with one of filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { oneOf: { name: ["Foo", "Bar"], @@ -707,7 +707,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with fuzzy filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { fuzzy: { name: "oo", @@ -721,7 +721,7 @@ describe("Google Sheets Integration", () => { it("should be able to find rows with range filter", async () => { const response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { range: { name: { @@ -750,7 +750,7 @@ describe("Google Sheets Integration", () => { }) let response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, @@ -759,7 +759,7 @@ describe("Google Sheets Integration", () => { while (response.hasNextPage) { response = await config.api.row.search(table._id!, { - sourceId: table._id!, + tableId: table._id!, query: { equal: { name: "Unique value!" } }, paginate: true, limit: 10, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index fa01a6cf13..c1685d8024 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,7 +14,7 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { searchInputMapping } from "./search/utils" -import { features, docIds } from "@budibase/backend-core" +import { features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" @@ -38,7 +38,8 @@ export async function search( ): Promise> { return await tracer.trace("search", async span => { span?.addTags({ - sourceId: options.sourceId, + tableId: options.tableId, + viewId: options.viewId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, @@ -76,20 +77,16 @@ export async function search( let source: Table | ViewV2 let table: Table - if (docIds.isTableId(options.sourceId)) { - source = await sdk.tables.getTable(options.sourceId) - table = source - options = searchInputMapping(source, options) - } else if (docIds.isViewId(options.sourceId)) { - source = await sdk.views.get(options.sourceId) - table = await sdk.tables.getTable(source.tableId) + if (options.viewId) { + source = await sdk.views.get(options.viewId) + table = await sdk.views.getTable(source) + options = searchInputMapping(table, options) + } else if (options.tableId) { + source = await sdk.tables.getTable(options.tableId) + table = source options = searchInputMapping(table, options) - - span.addTags({ - tableId: table._id, - }) } else { - throw new Error(`Invalid source ID: ${options.sourceId}`) + throw new Error(`Invalid source ID: ${options.viewId || options.tableId}`) } if (options.query) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index a41ae8dcda..925b472a48 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -200,7 +200,7 @@ export async function exportRows( } let result = await search( - { sourceId: table._id!, query: requestQuery, sort, sortOrder }, + { tableId: table._id!, query: requestQuery, sort, sortOrder }, table ) let rows: Row[] = [] diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index c9e2aba237..6617fc376c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -64,7 +64,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { let searchResponse = await sdk.rows.search({ - sourceId: tableId, + tableId, query, sort, sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 65cf4053d7..694c660625 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -19,8 +19,6 @@ export async function search( options: RowSearchParams, source: Table | ViewV2 ): Promise> { - const { sourceId } = options - let table: Table if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) @@ -31,7 +29,8 @@ export async function search( const { paginate, query } = options const params: RowSearchParams = { - sourceId: table._id!, + tableId: options.tableId, + viewId: options.viewId, sort: options.sort, sortOrder: options.sortOrder, sortType: options.sortType, @@ -59,7 +58,7 @@ export async function search( // Enrich search results with relationships if (response.rows && response.rows.length) { // enrich with global users if from users table - if (sourceId === InternalTables.USER_METADATA) { + if (table._id === InternalTables.USER_METADATA) { response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 194f2dd4e3..e7fd095865 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -122,7 +122,7 @@ describe.each([ it("querying by fields will always return data attribute columns", async () => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, fields: ["name", "age"], }) @@ -142,7 +142,7 @@ describe.each([ it("will decode _id in oneOf query", async () => { await config.doInContext(config.appId, async () => { const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: { oneOf: { _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], @@ -174,7 +174,7 @@ describe.each([ }, }) const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, }) expect(result.rows).toHaveLength(10) @@ -205,7 +205,7 @@ describe.each([ }, }) const result = await search({ - sourceId: table._id!, + tableId: table._id!, query: {}, fields: ["name", "age"], }) @@ -229,7 +229,7 @@ describe.each([ async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ - sourceId: table._id!, + tableId: table._id!, query: { $or: { conditions: [ diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index e3f241f15a..daa658455b 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - sourceId: tableId, + tableId: tableId, query: { equal: { "1:user": email, @@ -91,7 +91,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't error if no query supplied", () => { // @ts-expect-error - intentionally passing in a bad type - const output = searchInputMapping(col, { sourceId: tableId }) + const output = searchInputMapping(col, { tableId }) expect(output.query).toBeUndefined() }) } diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 2b6ff3a6c6..f81d56c082 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -10,7 +10,8 @@ export interface Aggregation { } export interface SearchParams { - sourceId?: string + tableId?: string + viewId?: string query?: SearchFilters paginate?: boolean bookmark?: string | number @@ -29,7 +30,7 @@ export interface SearchParams { // when searching for rows we want a more extensive search type that requires certain properties export interface RowSearchParams - extends WithRequired {} + extends WithRequired {} export interface SearchResponse { rows: T[] From 76453bd500dc3ab65fee06739843654467227b1c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Sep 2024 14:44:11 +0100 Subject: [PATCH 009/105] Fix many more search tests. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index c1685d8024..25d381b636 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -92,7 +92,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible) + ).filter(field => table.schema[field].visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) From 564e16fd5c5a231d485c1864fc29d0b30b027af3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Sep 2024 15:41:18 +0100 Subject: [PATCH 010/105] wip --- .../src/api/routes/tests/search.spec.ts | 2 +- .../src/sdk/app/rows/search/internal/sqs.ts | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d69e93cfd3..9cfd633ad0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2757,7 +2757,7 @@ describe.each([ }) }) - it.only("can filter by the row ID with limit 1", async () => { + it("can filter by the row ID with limit 1", async () => { await expectSearch({ query: { equal: { _id: row._id }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 5a9e1ddf24..438cff154c 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -45,6 +45,7 @@ import { import { dataFilters, helpers, + isInternalColumnName, PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" import { isSearchingByRowID } from "../utils" @@ -130,15 +131,22 @@ function cleanupFilters( // generate a map of all possible column names (these can be duplicated across tables // the map of them will always be the same const userColumnMap: Record = {} - allTables.forEach(table => - Object.keys(table.schema).forEach( - key => (userColumnMap[key] = mapToUserColumn(key)) - ) - ) + for (const table of allTables) { + for (const key of Object.keys(table.schema)) { + if (isInternalColumnName(key)) { + continue + } + userColumnMap[key] = mapToUserColumn(key) + } + } // update the keys of filters to manage user columns - const keyInAnyTable = (key: string): boolean => - allTables.some(table => table.schema[key]) + const keyInAnyTable = (key: string): boolean => { + if (isInternalColumnName(key)) { + return false + } + return allTables.some(table => table.schema[key]) + } const splitter = new dataFilters.ColumnSplitter(allTables) From 566af9e454e1440d2fc9ecae9b3926f8bf85b7e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Sep 2024 16:44:37 +0100 Subject: [PATCH 011/105] Fix bulk import to not modify the table schema. --- .../src/api/controllers/table/internal.ts | 22 +++++-------------- packages/server/src/utilities/schema.ts | 9 +++++++- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 6d1c67e800..1562054ab4 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -70,22 +70,10 @@ export async function bulkImport( ) { const table = await sdk.tables.getTable(ctx.params.tableId) const { rows, identifierFields } = ctx.request.body - await handleDataImport( - { - ...table, - schema: { - _id: { - name: "_id", - type: FieldType.STRING, - }, - ...table.schema, - }, - }, - { - importRows: rows, - identifierFields, - user: ctx.user, - } - ) + await handleDataImport(table, { + importRows: rows, + identifierFields, + user: ctx.user, + }) return table } diff --git a/packages/server/src/utilities/schema.ts b/packages/server/src/utilities/schema.ts index b1fbd7577a..cfdd0d753a 100644 --- a/packages/server/src/utilities/schema.ts +++ b/packages/server/src/utilities/schema.ts @@ -148,9 +148,16 @@ export function parse(rows: Rows, table: Table): Rows { Object.keys(row).forEach(columnName => { const columnData = row[columnName] + + if (columnName === "_id") { + parsedRow[columnName] = columnData + return + } + const schema = table.schema if (!(columnName in schema)) { - // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case + // Objects can be present in the row data but not in the schema, so make + // sure we don't proceed in such a case return } From efd677e16a59743f457a5ee7387373002a4839de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 25 Sep 2024 16:50:06 +0100 Subject: [PATCH 012/105] Most tests passing. --- .../server/src/api/controllers/table/internal.ts | 1 - .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- .../server/src/api/routes/tests/viewV2.spec.ts | 14 +++++++------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 1562054ab4..4286d51d3e 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -3,7 +3,6 @@ import { handleDataImport } from "./utils" import { BulkImportRequest, BulkImportResponse, - FieldType, RenameColumn, SaveTableRequest, SaveTableResponse, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 9cfd633ad0..090514250d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], + ["in-memory", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index d673e6ee5e..3bde9770cd 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -40,13 +40,13 @@ import { import sdk from "../../../sdk" describe.each([ - // ["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2384,7 +2384,7 @@ describe.each([ }) }) - describe("calculations", () => { + describe.skip("calculations", () => { let table: Table let rows: Row[] From 43265bf1ea9eb1c0886c780c85e8bd8a3a62e46b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 10:54:04 +0100 Subject: [PATCH 013/105] Rejig view calculation code to work with aggregates again. Broke some other tests in the process. --- .../src/api/controllers/row/utils/basic.ts | 71 ++++++++------- .../src/api/controllers/row/utils/sqlUtils.ts | 5 ++ .../src/api/controllers/row/utils/utils.ts | 50 ++++------- .../server/src/api/controllers/row/views.ts | 16 +--- .../src/api/routes/tests/viewV2.spec.ts | 16 ++-- packages/server/src/sdk/app/rows/search.ts | 3 - .../src/sdk/app/rows/search/internal/sqs.ts | 86 +++++++++++++------ packages/types/src/api/web/app/rows.ts | 1 - packages/types/src/sdk/row.ts | 1 - 9 files changed, 135 insertions(+), 114 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 20a1cd0742..9c86a165ab 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -5,7 +5,7 @@ import { Row, Table, JsonTypes, - Aggregation, + ViewV2, } from "@budibase/types" import { helpers, @@ -13,6 +13,7 @@ import { PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" import { generateRowIdField } from "../../../../integrations/utils" +import sdk from "../../../../sdk" function extractFieldValue({ row, @@ -85,22 +86,28 @@ function fixJsonTypes(row: Row, table: Table) { return row } -export function basicProcessing({ +export async function basicProcessing({ row, - table, + source, tables, isLinked, sqs, - aggregations, }: { row: Row - table: Table + source: Table | ViewV2 tables: Table[] isLinked: boolean sqs?: boolean - aggregations?: Aggregation[] -}): Row { +}): Promise { + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + const thisRow: Row = {} + // filter the row down to what is actually the row (not joined) for (let fieldName of Object.keys(table.schema)) { let value = extractFieldValue({ @@ -118,8 +125,10 @@ export function basicProcessing({ } } - for (let aggregation of aggregations || []) { - thisRow[aggregation.name] = row[aggregation.name] + if (sdk.views.isView(source)) { + for (const key of Object.keys(helpers.views.calculationFields(source))) { + thisRow[key] = row[key] + } } let columns: string[] = Object.keys(table.schema) @@ -163,28 +172,30 @@ export function basicProcessing({ thisRow[col] = array // make sure all of them have an _id const sortField = relatedTable.primaryDisplay || relatedTable.primary![0]! - thisRow[col] = (thisRow[col] as Row[]) - .map(relatedRow => - basicProcessing({ - row: relatedRow, - table: relatedTable, - tables, - isLinked: false, - sqs, - }) + thisRow[col] = ( + await Promise.all( + (thisRow[col] as Row[]).map(relatedRow => + basicProcessing({ + row: relatedRow, + source: relatedTable, + tables, + isLinked: false, + sqs, + }) + ) ) - .sort((a, b) => { - const aField = a?.[sortField], - bField = b?.[sortField] - if (!aField) { - return 1 - } else if (!bField) { - return -1 - } - return aField.localeCompare - ? aField.localeCompare(bField) - : aField - bField - }) + ).sort((a, b) => { + const aField = a?.[sortField], + bField = b?.[sortField] + if (!aField) { + return 1 + } else if (!bField) { + return -1 + } + return aField.localeCompare + ? aField.localeCompare(bField) + : aField - bField + }) } } return fixJsonTypes(thisRow, table) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 249bb43bbc..d51972b2c9 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -7,6 +7,7 @@ import { ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, + Row, Table, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" @@ -149,3 +150,7 @@ export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) { (DSPlusOperation.READ in resp[0] && resp[0].read === true) ) } + +export function isKnexRows(resp: DatasourcePlusQueryResponse): resp is Row[] { + return !isKnexEmptyReadResponse(resp) +} diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 45abd93930..b673106d26 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -2,7 +2,6 @@ import * as utils from "../../../../db/utils" import { docIds } from "@budibase/backend-core" import { - Aggregation, Ctx, DatasourcePlusQueryResponse, FieldType, @@ -15,7 +14,7 @@ import { processDates, processFormulas, } from "../../../../utilities/rowProcessor" -import { isKnexEmptyReadResponse } from "./sqlUtils" +import { isKnexRows } from "./sqlUtils" import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" @@ -97,7 +96,7 @@ export async function getTableFromSource(source: Table | ViewV2) { return source } -function fixBooleanFields({ row, table }: { row: Row; table: Table }) { +function fixBooleanFields(row: Row, table: Table) { for (let col of Object.values(table.schema)) { if (col.type === FieldType.BOOLEAN) { if (row[col.name] === 1) { @@ -115,53 +114,40 @@ export async function sqlOutputProcessing( source: Table | ViewV2, tables: Record, relationships: RelationshipsJson[], - opts?: { sqs?: boolean; aggregations?: Aggregation[] } + opts?: { sqs?: boolean } ): Promise { - if (isKnexEmptyReadResponse(rows)) { + if (!isKnexRows(rows)) { return [] } + let table: Table if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) } else { table = source } - let finalRows: { [key: string]: Row } = {} - for (let row of rows as Row[]) { - let rowId = row._id + + let processedRows: Row[] = [] + for (let row of rows) { if (opts?.sqs) { - rowId = getInternalRowId(row, table) - row._id = rowId - } else if (!rowId) { - rowId = generateIdForRow(row, table) - row._id = rowId + row._id = getInternalRowId(row, table) + } else if (row._id == null) { + row._id = generateIdForRow(row, table) } - const thisRow = basicProcessing({ + + row = await basicProcessing({ row, - table, + source, tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, - aggregations: opts?.aggregations, }) - if (thisRow._id == null) { - throw new Error("Unable to generate row ID for SQL rows") - } - - finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table }) + row = fixBooleanFields(row, table) + row = await processRelationshipFields(table, tables, row, relationships) + processedRows.push(row) } - // make sure all related rows are correct - let finalRowArray = [] - for (let row of Object.values(finalRows)) { - finalRowArray.push( - await processRelationshipFields(table, tables, row, relationships) - ) - } - - // process some additional types - finalRowArray = processDates(table, finalRowArray) - return finalRowArray + return processDates(table, processedRows) } export function isUserMetadataTable(tableId: string) { diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index fa75990136..68958da8e7 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -5,9 +5,8 @@ import { SearchViewRowRequest, SearchFilterKey, LogicalOperator, - Aggregation, } from "@budibase/types" -import { dataFilters, helpers } from "@budibase/shared-core" +import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" import { db, context, features } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" @@ -26,9 +25,6 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } - const viewFields = Object.entries(helpers.views.basicFields(view)) - .filter(([_, value]) => value.visible) - .map(([key]) => key) const { body } = ctx.request // Enrich saved query with ephemeral query params. @@ -73,25 +69,15 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) - const aggregations: Aggregation[] = Object.entries( - helpers.views.calculationFields(view) - ).map(([name, { field, calculationType }]) => ({ - name, - calculationType, - field, - })) - const result = await sdk.rows.search({ viewId: view.id, tableId: view.tableId, query: enrichedQuery, - fields: viewFields, ...getSortOptions(body, view), limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, - aggregations, }) result.rows.forEach(r => (r._viewId = view.id)) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 3bde9770cd..1d47769fb8 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -40,13 +40,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1653,7 +1653,7 @@ describe.each([ }) describe("search", () => { - it("returns empty rows from view when no schema is passed", async () => { + it.only("returns empty rows from view when no schema is passed", async () => { const rows = await Promise.all( Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) ) @@ -2384,7 +2384,7 @@ describe.each([ }) }) - describe.skip("calculations", () => { + describe("calculations", () => { let table: Table let rows: Row[] diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 25d381b636..cb7397689a 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -49,9 +49,6 @@ export async function search( paginate: options.paginate, fields: options.fields, countRows: options.countRows, - aggregations: options.aggregations - ?.map(a => `${a.field}:${a.calculationType}`) - .join(", "), }) options.query = dataFilters.cleanupQuery(options.query || {}) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 438cff154c..a01b1bc820 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -1,4 +1,5 @@ import { + Aggregation, Datasource, DocumentType, FieldType, @@ -58,11 +59,34 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) const MISSING_TABLE_REGX = new RegExp(`no such table: .+`) const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`) -function buildInternalFieldList( - table: Table, +async function buildInternalFieldList( + source: Table | ViewV2, tables: Table[], - opts?: { relationships?: RelationshipsJson[] } + opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] } ) { + const { relationships, allowedFields } = opts || {} + let schemaFields: string[] = [] + if (sdk.views.isView(source)) { + schemaFields = Object.keys(helpers.views.basicFields(source)).filter( + key => source.schema?.[key]?.visible !== false + ) + } else { + schemaFields = Object.keys(source.schema).filter( + key => source.schema[key].visible !== false + ) + } + + if (allowedFields) { + schemaFields = schemaFields.filter(field => allowedFields.includes(field)) + } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + let fieldList: string[] = [] const getJunctionFields = (relatedTable: Table, fields: string[]) => { const junctionFields: string[] = [] @@ -73,13 +97,18 @@ function buildInternalFieldList( }) return junctionFields } - fieldList = fieldList.concat( - PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`) - ) - for (let key of Object.keys(table.schema)) { + if (sdk.tables.isTable(source)) { + for (const key of PROTECTED_INTERNAL_COLUMNS) { + if (allowedFields && !allowedFields.includes(key)) { + continue + } + fieldList.push(`${table._id}.${key}`) + } + } + for (let key of schemaFields) { const col = table.schema[key] const isRelationship = col.type === FieldType.LINK - if (!opts?.relationships && isRelationship) { + if (!relationships && isRelationship) { continue } if (!isRelationship) { @@ -90,7 +119,9 @@ function buildInternalFieldList( if (!relatedTable) { continue } - const relatedFields = buildInternalFieldList(relatedTable, tables).concat( + const relatedFields = ( + await buildInternalFieldList(relatedTable, tables) + ).concat( getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"]) ) // break out of the loop if we have reached the max number of columns @@ -330,11 +361,20 @@ export async function search( documentType: DocumentType.ROW, } - if (options.aggregations) { - options.aggregations = options.aggregations.map(a => { - a.field = mapToUserColumn(a.field) - return a - }) + let aggregations: Aggregation[] = [] + if (sdk.views.isView(source)) { + const calculationFields = helpers.views.calculationFields(source) + for (const [key, field] of Object.entries(calculationFields)) { + if (options.fields && !options.fields.includes(key)) { + continue + } + + aggregations.push({ + name: key, + field: mapToUserColumn(field.field), + calculationType: field.calculationType, + }) + } } const request: QueryJson = { @@ -352,8 +392,11 @@ export async function search( columnPrefix: USER_COLUMN_PREFIX, }, resource: { - fields: buildInternalFieldList(table, allTables, { relationships }), - aggregations: options.aggregations, + fields: await buildInternalFieldList(source, allTables, { + relationships, + allowedFields: options.fields, + }), + aggregations, }, relationships, } @@ -400,7 +443,6 @@ export async function search( table, await sqlOutputProcessing(rows, source, allTablesMap, relationships, { sqs: true, - aggregations: options.aggregations, }) ) @@ -418,16 +460,12 @@ export async function search( let finalRows = await outputProcessing(source, processed, { preserveLinks: true, squash: true, - aggregations: options.aggregations, + aggregations, }) // check if we need to pick specific rows out if (options.fields) { - const fields = [ - ...options.fields, - ...PROTECTED_INTERNAL_COLUMNS, - ...(options.aggregations || []).map(a => a.name), - ] + const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] finalRows = finalRows.map((r: any) => pick(r, fields)) } @@ -450,7 +488,7 @@ export async function search( const msg = typeof err === "string" ? err : err.message if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) { await sdk.tables.sqs.syncDefinition() - return search(options, table, { retrying: true }) + return search(options, source, { retrying: true }) } // previously the internal table didn't error when a column didn't exist in search if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 5d49ac1812..ce6f6f672d 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -26,7 +26,6 @@ export interface SearchViewRowRequest | "paginate" | "query" | "countRows" - | "aggregations" > {} export interface SearchRowResponse { diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index f81d56c082..0b63e88229 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -25,7 +25,6 @@ export interface SearchParams { indexer?: () => Promise rows?: Row[] countRows?: boolean - aggregations?: Aggregation[] } // when searching for rows we want a more extensive search type that requires certain properties From 0ef633b87a9c3dfa9e08ef8e03471f39a0bb4e0b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 11:56:03 +0100 Subject: [PATCH 014/105] Fix viewV2.spec.ts for sqs --- .../src/api/routes/tests/viewV2.spec.ts | 24 +------------------ .../src/sdk/app/rows/search/internal/sqs.ts | 12 ++++++---- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d47769fb8..ba2e93afbf 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1653,7 +1653,7 @@ describe.each([ }) describe("search", () => { - it.only("returns empty rows from view when no schema is passed", async () => { + it("returns empty rows from view when no schema is passed", async () => { const rows = await Promise.all( Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) ) @@ -2197,28 +2197,6 @@ describe.each([ expect(response.rows).toHaveLength(0) }) - it("queries the row api passing the view fields only", async () => { - const searchSpy = jest.spyOn(sdk.rows, "search") - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - id: { visible: true }, - one: { visible: false }, - }, - }) - - await config.api.viewV2.search(view.id, { query: {} }) - expect(searchSpy).toHaveBeenCalledTimes(1) - - expect(searchSpy).toHaveBeenCalledWith( - expect.objectContaining({ - fields: ["id"], - }) - ) - }) - describe("foreign relationship columns", () => { let envCleanup: () => void beforeAll(() => { diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a01b1bc820..26e8f59303 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -463,11 +463,13 @@ export async function search( aggregations, }) - // check if we need to pick specific rows out - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] - finalRows = finalRows.map((r: any) => pick(r, fields)) - } + const visibleFields = + options.fields || + Object.keys(source.schema || {}).filter( + key => source.schema?.[key].visible !== false + ) + const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS] + finalRows = finalRows.map((r: any) => pick(r, allowedFields)) const response: SearchResponse = { rows: finalRows, From c4c524c6ff35921f036b4ef9fdb051bf58ecc15f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:22:10 +0100 Subject: [PATCH 015/105] viewV2.spec.ts passsing in full --- packages/backend-core/src/sql/sql.ts | 5 +- .../api/controllers/row/ExternalRequest.ts | 22 +++- .../src/api/controllers/row/utils/basic.ts | 32 ++--- .../src/api/controllers/row/utils/sqlUtils.ts | 41 +++++-- .../src/api/controllers/row/utils/utils.ts | 5 +- .../src/api/routes/tests/viewV2.spec.ts | 114 +++++++++--------- .../src/sdk/app/rows/search/external.ts | 11 +- .../sdk/app/rows/search/internal/lucene.ts | 11 +- .../src/sdk/app/rows/search/internal/sqs.ts | 1 - .../src/utilities/rowProcessor/index.ts | 9 +- 10 files changed, 151 insertions(+), 100 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index bfb9d9bfbe..b7bf5bc102 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -859,7 +859,7 @@ class InternalBuilder { } addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { - let { sort } = this.query + let { sort, resource } = this.query const primaryKey = this.table.primary const tableName = getTableName(this.table) const aliases = this.query.tableAliases @@ -896,7 +896,8 @@ class InternalBuilder { // add sorting by the primary key if the result isn't already sorted by it, // to make sure result is deterministic - if (!sort || sort[primaryKey[0]] === undefined) { + const hasAggregations = (resource?.aggregations?.length ?? 0) > 0 + if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) { query = query.orderBy(`${aliased}.${primaryKey[0]}`) } return query diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index a52d7abcd1..4ce326c35f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -1,5 +1,6 @@ import dayjs from "dayjs" import { + Aggregation, AutoFieldSubType, AutoReason, Datasource, @@ -47,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" import { makeExternalQuery } from "../../../integrations/base/query" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" export interface ManyRelationship { tableId?: string @@ -682,12 +683,26 @@ export class ExternalRequest { } } } + if ( operation === Operation.DELETE && (filters == null || Object.keys(filters).length === 0) ) { throw "Deletion must be filtered" } + + let aggregations: Aggregation[] = [] + if (sdk.views.isView(this.source)) { + const calculationFields = helpers.views.calculationFields(this.source) + for (const [key, field] of Object.entries(calculationFields)) { + aggregations.push({ + name: key, + field: field.field, + calculationType: field.calculationType, + }) + } + } + let json: QueryJson = { endpoint: { datasourceId: this.datasource._id!, @@ -697,10 +712,11 @@ export class ExternalRequest { resource: { // have to specify the fields to avoid column overlap (for SQL) fields: isSql - ? buildSqlFieldList(table, this.tables, { + ? await buildSqlFieldList(this.source, this.tables, { relationships: incRelationships, }) : [], + aggregations, }, filters, sort, @@ -748,7 +764,7 @@ export class ExternalRequest { } const output = await sqlOutputProcessing( response, - table, + this.source, this.tables, relationships ) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9c86a165ab..23565670ba 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -100,8 +100,10 @@ export async function basicProcessing({ sqs?: boolean }): Promise { let table: Table + let isCalculationView = false if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) + isCalculationView = helpers.views.isCalculationView(source) } else { table = source } @@ -132,20 +134,22 @@ export async function basicProcessing({ } let columns: string[] = Object.keys(table.schema) - if (!sqs) { - thisRow._id = generateIdForRow(row, table, isLinked) - thisRow.tableId = table._id - thisRow._rev = "rev" - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - } else { - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - thisRow[internalColumn] = extractFieldValue({ - row, - tableName: table._id!, - fieldName: internalColumn, - isLinked, - }) + if (!isCalculationView) { + if (!sqs) { + thisRow._id = generateIdForRow(row, table, isLinked) + thisRow.tableId = table._id + thisRow._rev = "rev" + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + } else { + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { + thisRow[internalColumn] = extractFieldValue({ + row, + tableName: table._id!, + fieldName: internalColumn, + isLinked, + }) + } } } for (let col of columns) { diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index d51972b2c9..36521b10c6 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -9,9 +9,12 @@ import { RelationshipsJson, Row, Table, + ViewV2, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" import { generateJunctionTableID } from "../../../../db/utils" +import sdk from "../../../../sdk" +import { helpers } from "@budibase/shared-core" type TableMap = Record @@ -109,11 +112,12 @@ export function buildInternalRelationships( * Creating the specific list of fields that we desire, and excluding the ones that are no use to us * is more performant and has the added benefit of protecting against this scenario. */ -export function buildSqlFieldList( - table: Table, +export async function buildSqlFieldList( + source: Table | ViewV2, tables: TableMap, opts?: { relationships: boolean } ) { + const { relationships } = opts || {} function extractRealFields(table: Table, existing: string[] = []) { return Object.entries(table.schema) .filter( @@ -124,22 +128,33 @@ export function buildSqlFieldList( ) .map(column => `${table.name}.${column[0]}`) } - let fields = extractRealFields(table) + + let fields: string[] = [] + if (sdk.views.isView(source)) { + fields = Object.keys(helpers.views.basicFields(source)).filter( + key => source.schema?.[key]?.visible !== false + ) + } else { + fields = extractRealFields(source) + } + + let table: Table + if (sdk.views.isView(source)) { + table = await sdk.views.getTable(source.id) + } else { + table = source + } + for (let field of Object.values(table.schema)) { - if ( - field.type !== FieldType.LINK || - !opts?.relationships || - !field.tableId - ) { + if (field.type !== FieldType.LINK || !relationships || !field.tableId) { continue } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - const linkTable = tables[linkTableName] - if (linkTable) { - const linkedFields = extractRealFields(linkTable, fields) - fields = fields.concat(linkedFields) + const { tableName } = breakExternalTableId(field.tableId) + if (tables[tableName]) { + fields = fields.concat(extractRealFields(tables[tableName], fields)) } } + return fields } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index b673106d26..4188fcced3 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -19,6 +19,7 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" +import { helpers } from "@budibase/shared-core" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -121,8 +122,10 @@ export async function sqlOutputProcessing( } let table: Table + let isCalculationView = false if (sdk.views.isView(source)) { table = await sdk.views.getTable(source.id) + isCalculationView = helpers.views.isCalculationView(source) } else { table = source } @@ -131,7 +134,7 @@ export async function sqlOutputProcessing( for (let row of rows) { if (opts?.sqs) { row._id = getInternalRowId(row, table) - } else if (row._id == null) { + } else if (row._id == null && !isCalculationView) { row._id = generateIdForRow(row, table) } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index ba2e93afbf..883b25aca1 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,16 +37,15 @@ import { setEnv as setCoreEnv, env, } from "@budibase/backend-core" -import sdk from "../../../sdk" describe.each([ - // ["lucene", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2362,63 +2361,70 @@ describe.each([ }) }) - describe("calculations", () => { - let table: Table - let rows: Row[] + !isLucene && + describe("calculations", () => { + let table: Table + let rows: Row[] - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, }, - price: { - type: FieldType.NUMBER, - name: "price", + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", }, }, }) - ) - rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, { - quantity: generator.natural({ min: 1, max: 10 }), - price: generator.natural({ min: 1, max: 10 }), - }) + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) ) - ) - }) - it("should be able to search by calculations", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - "Quantity Sum": { - visible: true, - calculationType: CalculationType.SUM, - field: "quantity", - }, - }, + // Calculation views do not return rows that can be linked back to + // the source table, and so should not have an _id field. + for (const row of response.rows) { + expect("_id" in row).toBe(false) + } }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), - }), - ]) - ) }) - }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 925b472a48..fa8961461b 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -127,10 +127,13 @@ export async function search( } } - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS] - processed = processed.map((r: any) => pick(r, fields)) - } + const visibleFields = + options.fields || + Object.keys(source.schema || {}).filter( + key => source.schema?.[key].visible !== false + ) + const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS] + processed = processed.map((r: any) => pick(r, allowedFields)) // need wrapper object for bookmarks etc when paginating const response: SearchResponse = { rows: processed, hasNextPage } diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts index 694c660625..953fb90c1f 100644 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ b/packages/server/src/sdk/app/rows/search/internal/lucene.ts @@ -62,10 +62,13 @@ export async function search( response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) } - if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] - response.rows = response.rows.map((r: any) => pick(r, fields)) - } + const visibleFields = + options.fields || + Object.keys(source.schema || {}).filter( + key => source.schema?.[key].visible !== false + ) + const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS] + response.rows = response.rows.map((r: any) => pick(r, allowedFields)) response.rows = await outputProcessing(source, response.rows, { squash: true, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 26e8f59303..c185ef18dd 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -460,7 +460,6 @@ export async function search( let finalRows = await outputProcessing(source, processed, { preserveLinks: true, squash: true, - aggregations, }) const visibleFields = diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 24c8d11bd1..818c98a84e 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,7 +11,6 @@ import { import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { - Aggregation, AutoFieldSubType, FieldType, IdentityType, @@ -263,7 +262,6 @@ export async function outputProcessing( preserveLinks?: boolean fromRow?: Row skipBBReferences?: boolean - aggregations?: Aggregation[] } = { squash: true, preserveLinks: false, @@ -411,8 +409,11 @@ export async function outputProcessing( f.toLowerCase() ) - for (const aggregation of opts.aggregations || []) { - fields.push(aggregation.name.toLowerCase()) + if (sdk.views.isView(source)) { + const aggregations = helpers.views.calculationFields(source) + for (const key of Object.keys(aggregations)) { + fields.push(key.toLowerCase()) + } } for (const row of enriched) { From 7c6c03c80b2ad15944b6c830f1abc6fd213d7c8e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:32:21 +0100 Subject: [PATCH 016/105] Rename ViewUIFieldMetadata -> ViewFieldMetadata to match master. --- packages/server/src/api/controllers/view/viewsV2.ts | 12 ++++++------ packages/server/src/api/routes/tests/viewV2.spec.ts | 4 ++-- packages/server/src/db/linkedRows/index.ts | 4 ++-- packages/shared-core/src/helpers/views.ts | 10 +++++----- packages/types/src/documents/app/view.ts | 10 +++++----- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index fddcef97a2..7f6f638541 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -7,17 +7,17 @@ import { ViewResponse, ViewResponseEnriched, ViewV2, - BasicViewUIFieldMetadata, + BasicViewFieldMetadata, ViewCalculationFieldMetadata, RelationSchemaField, - ViewUIFieldMetadata, + ViewFieldMetadata, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" function stripUnknownFields( - field: BasicViewUIFieldMetadata -): RequiredKeys { + field: BasicViewFieldMetadata +): RequiredKeys { if (helpers.views.isCalculationField(field)) { const strippedField: RequiredKeys = { order: field.order, @@ -31,7 +31,7 @@ function stripUnknownFields( } return strippedField } else { - const strippedField: RequiredKeys = { + const strippedField: RequiredKeys = { order: field.order, width: field.width, visible: field.visible, @@ -83,7 +83,7 @@ async function parseSchema(view: CreateViewRequest) { p[fieldName] = fieldSchema return p - }, {} as Record>) + }, {} as Record>) return finalViewSchema } diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 883b25aca1..3712efa5a4 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,7 +22,7 @@ import { RelationshipType, TableSchema, RenameColumn, - ViewUIFieldMetadata, + ViewFieldMetadata, FeatureFlag, BBReferenceFieldSubType, } from "@budibase/types" @@ -1154,7 +1154,7 @@ describe.each([ const createView = async ( tableId: string, - schema: Record + schema: Record ) => await config.api.viewV2.create({ name: generator.guid(), diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index ed660288c8..41cbc5b9c1 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -20,7 +20,7 @@ import { Row, Table, TableSchema, - ViewUIFieldMetadata, + ViewFieldMetadata, ViewV2, } from "@budibase/types" import sdk from "../../sdk" @@ -263,7 +263,7 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} + let viewSchema: Record = {} if (sdk.views.isView(source)) { if (helpers.views.isCalculationView(source)) { return enriched diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts index c65bc4882d..f41c66adc8 100644 --- a/packages/shared-core/src/helpers/views.ts +++ b/packages/shared-core/src/helpers/views.ts @@ -1,20 +1,20 @@ import { - BasicViewUIFieldMetadata, + BasicViewFieldMetadata, ViewCalculationFieldMetadata, - ViewUIFieldMetadata, + ViewFieldMetadata, ViewV2, } from "@budibase/types" import { pickBy } from "lodash" export function isCalculationField( - field: ViewUIFieldMetadata + field: ViewFieldMetadata ): field is ViewCalculationFieldMetadata { return "calculationType" in field } export function isBasicViewField( - field: ViewUIFieldMetadata -): field is BasicViewUIFieldMetadata { + field: ViewFieldMetadata +): field is BasicViewFieldMetadata { return !isCalculationField(field) } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 74b8f61f59..a957564039 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -33,7 +33,7 @@ export interface View { groupBy?: string } -export interface BasicViewUIFieldMetadata extends UIFieldMetadata { +export interface BasicViewFieldMetadata extends UIFieldMetadata { readonly?: boolean columns?: Record } @@ -42,13 +42,13 @@ export interface RelationSchemaField extends UIFieldMetadata { readonly?: boolean } -export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata { +export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata { calculationType: CalculationType field: string } -export type ViewUIFieldMetadata = - | BasicViewUIFieldMetadata +export type ViewFieldMetadata = + | BasicViewFieldMetadata | ViewCalculationFieldMetadata export enum CalculationType { @@ -71,7 +71,7 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: Record } export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema From 25a2e02a90acf6b07b71dce4cde62d4899ce7cdd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:40:25 +0100 Subject: [PATCH 017/105] Remove needless table copy. --- packages/server/src/api/controllers/row/internal.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index f5cb42f81d..bb9de6ce52 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -53,11 +53,8 @@ export async function patch(ctx: UserCtx) { combinedRow[key] = inputs[key] } - // need to copy the table so it can be differenced on way out - const tableClone = cloneDeep(table) - // this returns the table and row incase they have been updated - let row = await inputProcessing(ctx.user?._id, tableClone, combinedRow) + let row = await inputProcessing(ctx.user?._id, source, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, source, From 26a27ff70f1a0e9a363ce9e9d19cae123f1e7776 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:48:44 +0100 Subject: [PATCH 018/105] Remove needless table copy. --- packages/server/src/sdk/app/rows/internal.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index 13054f43f4..997af3c907 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -34,10 +34,7 @@ export async function save( inputs._id = db.generateRowID(inputs.tableId) } - // need to copy the table so it can be differenced on way out - const sourceClone = cloneDeep(source) - - let row = await inputProcessing(userId, sourceClone, inputs) + let row = await inputProcessing(userId, source, inputs) const validateResult = await sdk.rows.utils.validate({ row, From ec6fa5f79b2dda85dbb7043291fecf6bc59ec3f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:50:49 +0100 Subject: [PATCH 019/105] Return SQS error to prevoius state. --- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index c185ef18dd..f0e47ce2b7 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -495,6 +495,6 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } - throw err + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } From ae8a8645660518404dba217013627c126c2783a5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:51:21 +0100 Subject: [PATCH 020/105] Collapse duplicated key names. --- packages/server/src/sdk/app/rows/search/tests/utils.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts index daa658455b..33594a621c 100644 --- a/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/utils.spec.ts @@ -48,7 +48,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should be able to map ro_ to global user IDs", () => { const params: RowSearchParams = { - tableId: tableId, + tableId, query: { equal: { "1:user": userMedataId, @@ -61,7 +61,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("should handle array of user IDs", () => { const params: RowSearchParams = { - tableId: tableId, + tableId, query: { oneOf: { "1:user": [userMedataId, globalUserId], @@ -78,7 +78,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])( it("shouldn't change any other input", () => { const email = "test@example.com" const params: RowSearchParams = { - tableId: tableId, + tableId, query: { equal: { "1:user": email, From 743140eeaee5ea0d6b2269470e98d63411174d7f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 15:55:37 +0100 Subject: [PATCH 021/105] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index afe53b4be5..6c82625fdc 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit afe53b4be55ac6f93dc9486596835412b2673f90 +Subproject commit 6c82625fdca3318e9fdbb61b8c2e9e78e5f8ace9 From aa738659aea8b8043a3dd834072448446b261f19 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 16:21:34 +0100 Subject: [PATCH 022/105] Respond to PR feedback. --- .../src/api/controllers/row/utils/basic.ts | 30 +++++++++---------- packages/server/src/sdk/app/rows/internal.ts | 1 - 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 23565670ba..a7aa82e1f2 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -134,22 +134,20 @@ export async function basicProcessing({ } let columns: string[] = Object.keys(table.schema) - if (!isCalculationView) { - if (!sqs) { - thisRow._id = generateIdForRow(row, table, isLinked) - thisRow.tableId = table._id - thisRow._rev = "rev" - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - } else { - columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) - for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - thisRow[internalColumn] = extractFieldValue({ - row, - tableName: table._id!, - fieldName: internalColumn, - isLinked, - }) - } + if (!sqs && !isCalculationView) { + thisRow._id = generateIdForRow(row, table, isLinked) + thisRow.tableId = table._id + thisRow._rev = "rev" + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + } else if (!isCalculationView) { + columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) + for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { + thisRow[internalColumn] = extractFieldValue({ + row, + tableName: table._id!, + fieldName: internalColumn, + isLinked, + }) } } for (let col of columns) { diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts index 997af3c907..9306609132 100644 --- a/packages/server/src/sdk/app/rows/internal.ts +++ b/packages/server/src/sdk/app/rows/internal.ts @@ -1,7 +1,6 @@ import { context, db } from "@budibase/backend-core" import { Row, Table, ViewV2 } from "@budibase/types" import sdk from "../../../sdk" -import cloneDeep from "lodash/fp/cloneDeep" import { finaliseRow } from "../../../api/controllers/row/staticFormula" import { inputProcessing, From 8dd21e55929ec1968cf89ec9fcb36ac930513e02 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 17:06:51 +0100 Subject: [PATCH 023/105] Wider check on fields == null. --- packages/server/src/sdk/app/rows/queryUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index a88763215e..7ef776a989 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -110,7 +110,7 @@ export const getQueryableFields = async ( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] - if (fields === undefined) { + if (fields == null) { fields = Object.keys(table.schema) } result.push(...(await extractTableFields(table, fields, [table._id!]))) From d7ffdf02c27c9fcabc6f85c094471ee2e36834e4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 17:10:53 +0100 Subject: [PATCH 024/105] Update isTable and isView to depend on the ID format. --- packages/server/src/sdk/app/tables/utils.ts | 3 ++- packages/server/src/sdk/app/views/index.ts | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/src/sdk/app/tables/utils.ts b/packages/server/src/sdk/app/tables/utils.ts index 7a8096fb0a..96499db00a 100644 --- a/packages/server/src/sdk/app/tables/utils.ts +++ b/packages/server/src/sdk/app/tables/utils.ts @@ -1,5 +1,6 @@ import { Table, TableSourceType } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" +import { docIds } from "@budibase/backend-core" export function isExternal(opts: { table?: Table; tableId?: string }): boolean { if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) { @@ -11,5 +12,5 @@ export function isExternal(opts: { table?: Table; tableId?: string }): boolean { } export function isTable(table: any): table is Table { - return table.type === "table" + return table._id && docIds.isTableId(table._id) } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4251383712..4aea36fa2a 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -9,7 +9,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { context, HTTPError } from "@budibase/backend-core" +import { context, docIds, HTTPError } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -53,9 +53,7 @@ export async function getTable(view: string | ViewV2): Promise
{ } export function isView(view: any): view is ViewV2 { - return ( - view.version === 2 && "id" in view && "tableId" in view && "name" in view - ) + return view._id && docIds.isViewId(view._id) && view.version === 2 } async function guardCalculationViewSchema( From 559988e01134258e1954aeae5efa1326defea35f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 17:11:50 +0100 Subject: [PATCH 025/105] Correct error message. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index cb7397689a..809bd73d1f 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -83,7 +83,7 @@ export async function search( table = source options = searchInputMapping(table, options) } else { - throw new Error(`Invalid source ID: ${options.viewId || options.tableId}`) + throw new Error(`Must supply either a view ID or a table ID`) } if (options.query) { From 264b10f3f398a5e66189f5503f8802b0cca0f141 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 26 Sep 2024 17:22:11 +0100 Subject: [PATCH 026/105] Fix isView. --- packages/server/src/sdk/app/views/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4aea36fa2a..cd0fdf077e 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -53,7 +53,7 @@ export async function getTable(view: string | ViewV2): Promise
{ } export function isView(view: any): view is ViewV2 { - return view._id && docIds.isViewId(view._id) && view.version === 2 + return view.id && docIds.isViewId(view.id) && view.version === 2 } async function guardCalculationViewSchema( From 5919f1becdaf5745f5662eba202219bb0151e367 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 27 Sep 2024 09:57:56 +0100 Subject: [PATCH 027/105] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 6c82625fdc..ef8690c955 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 6c82625fdca3318e9fdbb61b8c2e9e78e5f8ace9 +Subproject commit ef8690c955cb83768f21770ece72d68cccddc8a9 From b2545a30e1850a981394c627fd6520ef1d05a03a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Sep 2024 11:51:28 +0100 Subject: [PATCH 028/105] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index ef8690c955..3e24f6293f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ef8690c955cb83768f21770ece72d68cccddc8a9 +Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0 From 5d319768359953035226f53d9ac5ad5f027dc8f6 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 30 Sep 2024 13:07:32 +0100 Subject: [PATCH 029/105] updated automation thread to use ids and test --- .../tests/scenarios/branching.spec.ts | 57 +++++++++++------ .../tests/utilities/AutomationTestBuilder.ts | 61 ++++++++++++------- .../server/src/definitions/automations.ts | 1 + packages/server/src/threads/automation.ts | 18 ++++-- 4 files changed, 91 insertions(+), 46 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts index ae89fc18b5..76e04afdd3 100644 --- a/packages/server/src/automations/tests/scenarios/branching.spec.ts +++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts @@ -17,44 +17,65 @@ describe("Branching automations", () => { afterAll(setup.afterAll) it("should run a multiple nested branching automation", async () => { + const firstLogId = "11111111-1111-1111-1111-111111111111" + const branch1LogId = "22222222-2222-2222-2222-222222222222" + const branch2LogId = "33333333-3333-3333-3333-333333333333" + const branch2Id = "44444444-4444-4444-4444-444444444444" + const builder = createAutomationBuilder({ name: "Test Trigger with Loop and Create Row", }) const results = await builder .appAction({ fields: {} }) - .serverLog({ text: "Starting automation" }) + .serverLog( + { text: "Starting automation" }, + { stepName: "FirstLog", id: firstLogId } + ) .branch({ topLevelBranch1: { steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1" }).branch({ - branch1: { - steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1.1" }), - condition: { - equal: { "{{steps.1.success}}": true }, + stepBuilder + .serverLog( + { text: "Branch 1" }, + { id: "66666666-6666-6666-6666-666666666666" } + ) + .branch({ + branch1: { + steps: stepBuilder => + stepBuilder.serverLog( + { text: "Branch 1.1" }, + { id: branch1LogId } + ), + condition: { + equal: { [`{{ steps.${firstLogId}.success }}`]: true }, + }, }, - }, - branch2: { - steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 1.2" }), - condition: { - equal: { "{{steps.1.success}}": false }, + branch2: { + steps: stepBuilder => + stepBuilder.serverLog( + { text: "Branch 1.2" }, + { id: branch2LogId } + ), + condition: { + equal: { [`{{ steps.${firstLogId}.success }}`]: false }, + }, }, - }, - }), + }), condition: { - equal: { "{{steps.1.success}}": true }, + equal: { [`{{ steps.${firstLogId}.success }}`]: true }, }, }, topLevelBranch2: { - steps: stepBuilder => stepBuilder.serverLog({ text: "Branch 2" }), + steps: stepBuilder => + stepBuilder.serverLog({ text: "Branch 2" }, { id: branch2Id }), condition: { - equal: { "{{steps.1.success}}": false }, + equal: { [`{{ steps.${firstLogId}.success }}`]: false }, }, }, }) .run() + expect(results.steps[3].outputs.status).toContain("branch1 branch taken") expect(results.steps[4].outputs.message).toContain("Branch 1.1") }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 2269f075b2..6af18cd27e 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -64,18 +64,18 @@ class BaseStepBuilder { stepId: TStep, stepSchema: Omit, inputs: AutomationStepInputs, - stepName?: string + opts?: { stepName?: string; id?: string } ): this { - const id = uuidv4() + const id = opts?.id || uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, id, stepId, - name: stepName || stepSchema.name, + name: opts?.stepName || stepSchema.name, }) - if (stepName) { - this.stepNames[id] = stepName + if (opts?.stepName) { + this.stepNames[id] = opts.stepName } return this } @@ -95,7 +95,6 @@ class BaseStepBuilder { }) branchStepInputs.children![key] = stepBuilder.build() }) - const branchStep: AutomationStep = { ...definition, id: uuidv4(), @@ -106,80 +105,98 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this { + createRow( + inputs: CreateRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs, - opts?.stepName + opts ) } - updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this { + updateRow( + inputs: UpdateRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs, - opts?.stepName + opts ) } - deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this { + deleteRow( + inputs: DeleteRowStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs, - opts?.stepName + opts ) } sendSmtpEmail( inputs: SmtpEmailStepInputs, - opts?: { stepName?: string } + opts?: { stepName?: string; id?: string } ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs, - opts?.stepName + opts ) } executeQuery( inputs: ExecuteQueryStepInputs, - opts?: { stepName?: string } + opts?: { stepName?: string; id?: string } ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs, - opts?.stepName + opts ) } - queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this { + queryRows( + inputs: QueryRowsStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs, - opts?.stepName + opts ) } - loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this { + loop( + inputs: LoopStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, inputs, - opts?.stepName + opts ) } - serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this { + serverLog( + input: ServerLogStepInputs, + opts?: { stepName?: string; id?: string } + ): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, input, - opts?.stepName + opts ) } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 6488e604e9..44758b727b 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,6 +15,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + stepsById?: Record stepsByName?: Record env?: Record trigger: any diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index e2a5a1c192..7788744ea2 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -91,6 +91,7 @@ class Orchestrator { // step zero is never used as the template string is zero indexed for customer facing this.context = { steps: [{}], + stepsById: {}, stepsByName: {}, trigger: triggerOutput, } @@ -457,6 +458,7 @@ class Orchestrator { inputs: steps[stepToLoopIndex].inputs, }) + this.context.stepsById![steps[stepToLoopIndex].id] = tempOutput const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id this.context.stepsByName![stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput @@ -517,7 +519,10 @@ class Orchestrator { Object.entries(filter).forEach(([_, value]) => { Object.entries(value).forEach(([field, _]) => { const updatedField = field.replace("{{", "{{ literal ") - const fromContext = processStringSync(updatedField, this.context) + const fromContext = processStringSync( + updatedField, + this.processContext(this.context) + ) toFilter[field] = fromContext }) }) @@ -563,9 +568,9 @@ class Orchestrator { } const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await this.addContextAndProcess( + let inputs = await processObject( originalStepInput, - this.context + this.processContext(this.context) ) inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) @@ -594,16 +599,16 @@ class Orchestrator { return null } - private async addContextAndProcess(inputs: any, context: any) { + private processContext(context: AutomationContext) { const processContext = { ...context, steps: { ...context.steps, + ...context.stepsById, ...context.stepsByName, }, } - - return processObject(inputs, processContext) + return processContext } private handleStepOutput( @@ -623,6 +628,7 @@ class Orchestrator { } else { this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) this.context.steps[this.context.steps.length] = outputs + this.context.stepsById![step.id] = outputs const stepName = step.name || step.id this.context.stepsByName![stepName] = outputs } From e281250569a9bcd67d9d2e998959682018097b3e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 30 Sep 2024 14:00:12 +0100 Subject: [PATCH 030/105] ai cron helper E2E --- packages/builder/package.json | 1 + .../SetupPanel/AutomationBlockSetup.svelte | 2 +- .../automation/SetupPanel/CronBuilder.svelte | 179 ++++++++--- .../SetupPanel/test/CronBuilder.spec.js | 0 packages/frontend-core/src/api/ai.js | 11 + packages/frontend-core/src/api/index.js | 2 + packages/server/src/api/routes/index.ts | 2 + packages/shared-core/src/helpers/cron.ts | 14 + yarn.lock | 295 ++++++++++-------- 9 files changed, 339 insertions(+), 167 deletions(-) create mode 100644 packages/builder/src/components/automation/SetupPanel/test/CronBuilder.spec.js create mode 100644 packages/frontend-core/src/api/ai.js diff --git a/packages/builder/package.json b/packages/builder/package.json index f9e6becbab..aec0b509f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -67,6 +67,7 @@ "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", "codemirror": "^5.65.16", + "cron-parser": "^4.9.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index aceb980786..7f8a68bf37 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -1048,7 +1048,7 @@ {:else if value.customType === "cron"} onChange({ [key]: e.detail })} - value={inputData[key]} + cronExpression={inputData[key]} /> {:else if value.customType === "automationFields"} - import { Button, Select, Input, Label } from "@budibase/bbui" + import { Button, Select, Icon, InlineAlert, Input, Label, Layout, Popover } from "@budibase/bbui" import { onMount, createEventDispatcher } from "svelte" import { flags } from "stores/builder" + import { licensing } from "stores/portal" + import { API } from "api" + import { helpers, REBOOT_CRON } from "@budibase/shared-core" const dispatch = createEventDispatcher() - export let value + export let cronExpression + let error + let nextExecutions + // AI prompt + let aiCronPrompt = "" + let loadingAICronExpression = false + + $: aiEnabled = $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: { - const exists = CRON_EXPRESSIONS.some(cron => cron.value === value) - const customIndex = CRON_EXPRESSIONS.findIndex( - cron => cron.label === "Custom" - ) - - if (!exists && customIndex === -1) { - CRON_EXPRESSIONS[0] = { label: "Custom", value: value } - } else if (exists && customIndex !== -1) { - CRON_EXPRESSIONS.splice(customIndex, 1) + if (cronExpression) { + try { + nextExecutions = helpers.cron.getNextExecutionDates(cronExpression).join("\n") + } catch (err) { + nextExecutions = null + } } } const onChange = e => { - if (value !== REBOOT_CRON) { + if (e.detail !== REBOOT_CRON) { error = helpers.cron.validate(e.detail).err } - if (e.detail === value || error) { + if (e.detail === cronExpression || error) { return } - value = e.detail + cronExpression = e.detail dispatch("change", e.detail) } + const updatePreset = e => { + aiCronPrompt = "" + onChange(e) + } + + const updateCronExpression = e => { + aiCronPrompt = "" + cronExpression = null + onChange(e) + } + let touched = false - let presets = false const CRON_EXPRESSIONS = [ { @@ -64,45 +81,129 @@ }) } }) + + async function generateAICronExpression() { + loadingAICronExpression = true + // make the API call to generate the cron expression + const response = await API.generateCronExpression({ prompt: aiCronPrompt }) + // return it and set it in the field + cronExpression = response.message + dispatch("change", response.message) + loadingAICronExpression = false + } -
+ + + {#if aiCronPrompt} +
+ +
+ {/if} +
+ {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} {/if} -
- - {#if presets} - =0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20031,33 +20045,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21620,7 +21614,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21630,14 +21624,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22166,6 +22152,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22940,33 +22934,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22976,11 +22947,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From f28cb1badbb503e548d4faf871bb789bba9f0d05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 15:18:15 +0100 Subject: [PATCH 045/105] Another slight change. --- packages/server/src/sdk/app/rows/search/utils.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 8d96c2f8f9..1dba420a28 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -105,11 +105,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } } } - return filters + return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters( - dataFilters.recurseLogicalOperators(options.query, checkFilters) - ) + options.query = checkFilters(options.query) return options } From 28bb3215115aada6a10c50202eae175916c1343d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Sep 2024 15:36:49 +0100 Subject: [PATCH 046/105] Set view permissions to explicit roles from the parent table --- .../server/src/api/controllers/permission.ts | 108 ++---------------- .../src/api/routes/tests/permissions.spec.ts | 15 +++ .../src/api/routes/tests/rowAction.spec.ts | 9 ++ .../src/api/routes/tests/viewV2.spec.ts | 5 + .../server/src/sdk/app/permissions/index.ts | 98 +++++++++++++++- packages/server/src/sdk/app/views/index.ts | 29 ++++- 6 files changed, 162 insertions(+), 102 deletions(-) diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index 66a3254348..c7afb6a351 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,9 +1,7 @@ import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, - Database, Role, - PermissionLevel, GetResourcePermsResponse, ResourcePermissionInfo, GetDependantResourcesResponse, @@ -12,107 +10,15 @@ import { RemovePermissionRequest, RemovePermissionResponse, } from "@budibase/types" -import { getRoleParams } from "../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../utilities/security" -import { removeFromArray } from "../../utilities" import sdk from "../../sdk" - -const enum PermissionUpdateType { - REMOVE = "remove", - ADD = "add", -} +import { PermissionUpdateType } from "../../sdk/app/permissions" const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS -// utility function to stop this repetition - permissions always stored under roles -async function getAllDBRoles(db: Database) { - const body = await db.allDocs( - getRoleParams(null, { - include_docs: true, - }) - ) - return body.rows.map(row => row.doc!) -} - -async function updatePermissionOnRole( - { - roleId, - resourceId, - level, - }: { roleId: string; resourceId: string; level: PermissionLevel }, - updateType: PermissionUpdateType -) { - const db = context.getAppDB() - const remove = updateType === PermissionUpdateType.REMOVE - const isABuiltin = roles.isBuiltin(roleId) - const dbRoleId = roles.getDBRoleID(roleId) - const dbRoles = await getAllDBRoles(db) - const docUpdates: Role[] = [] - - // the permission is for a built in, make sure it exists - if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { - const builtin = roles.getBuiltinRoles()[roleId] - builtin._id = roles.getDBRoleID(builtin._id!) - dbRoles.push(builtin) - } - - // now try to find any roles which need updated, e.g. removing the - // resource from another role and then adding to the new role - for (let role of dbRoles) { - let updated = false - const rolePermissions: Record = role.permissions - ? role.permissions - : {} - // make sure its an array, also handle migrating - if ( - !rolePermissions[resourceId] || - !Array.isArray(rolePermissions[resourceId]) - ) { - rolePermissions[resourceId] = - typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId] as unknown as PermissionLevel] - : [] - } - // handle the removal/updating the role which has this permission first - // the updating (role._id !== dbRoleId) is required because a resource/level can - // only be permitted in a single role (this reduces hierarchy confusion and simplifies - // the general UI for this, rather than needing to show everywhere it is used) - if ( - (role._id !== dbRoleId || remove) && - rolePermissions[resourceId].indexOf(level) !== -1 - ) { - removeFromArray(rolePermissions[resourceId], level) - updated = true - } - // handle the adding, we're on the correct role, at it to this - if (!remove && role._id === dbRoleId) { - const set = new Set(rolePermissions[resourceId]) - rolePermissions[resourceId] = [...set.add(level)] - updated = true - } - // handle the update, add it to bulk docs to perform at end - if (updated) { - role.permissions = rolePermissions - docUpdates.push(role) - } - } - - const response = await db.bulkDocs(docUpdates) - return response.map(resp => { - const version = docUpdates.find(role => role._id === resp.id)?.version - const _id = roles.getExternalRoleID(resp.id, version) - return { - _id, - rev: resp.rev, - error: resp.error, - reason: resp.reason, - } - }) -} - export function fetchBuiltin(ctx: UserCtx) { ctx.body = Object.values(permissions.getBuiltinPermissions()) } @@ -124,7 +30,7 @@ export function fetchLevels(ctx: UserCtx) { export async function fetch(ctx: UserCtx) { const db = context.getAppDB() - const dbRoles: Role[] = await getAllDBRoles(db) + const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db) let permissions: any = {} // create an object with structure role ID -> resource ID -> level for (let role of dbRoles) { @@ -186,12 +92,18 @@ export async function getDependantResources( export async function addPermission(ctx: UserCtx) { const params: AddPermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.ADD + ) } export async function removePermission( ctx: UserCtx ) { const params: RemovePermissionRequest = ctx.params - ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE) + ctx.body = await sdk.permissions.updatePermissionOnRole( + params, + PermissionUpdateType.REMOVE + ) } diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 0f059998ae..b148d6fde1 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -125,6 +125,13 @@ describe("/permission", () => { }) it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => { + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: view.id, + level: PermissionLevel.READ, + }) + // replicate changes before checking permissions await config.publish() @@ -138,6 +145,14 @@ describe("/permission", () => { resourceId: table._id, level: PermissionLevel.READ, }) + + // Make view inherit table permissions. Needed for backwards compatibility with existing views. + await config.api.permission.revoke({ + roleId: STD_ROLE_ID, + resourceId: view.id, + level: PermissionLevel.READ, + }) + // replicate changes before checking permissions await config.publish() diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index ef7d2afbba..4fe248984a 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -826,11 +826,20 @@ describe("/rowsActions", () => { ) ).id + // Allow row action on view await config.api.rowAction.setViewPermission( tableId, viewId, rowAction.id ) + + // Delete explicit view permissions so they inherit table permissions + await config.api.permission.revoke({ + roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission + level: PermissionLevel.READ, + resourceId: viewId, + }) + return { permissionResource: tableId, triggerResouce: viewId } }, ], diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 3712efa5a4..aab846e704 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2460,6 +2460,11 @@ describe.each([ level: PermissionLevel.READ, resourceId: table._id!, }) + await config.api.permission.revoke({ + roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission + level: PermissionLevel.READ, + resourceId: view.id, + }) await config.publish() const response = await config.api.viewV2.publicSearch(view.id) diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index d5e4aefe3a..2c3c0af95b 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,22 +1,30 @@ -import { db, docIds, roles } from "@budibase/backend-core" +import { db, roles, context, docIds } from "@budibase/backend-core" import { PermissionLevel, PermissionSource, VirtualDocumentType, + Role, + Database, } from "@budibase/types" -import { extractViewInfoFromID } from "../../../db/utils" +import { extractViewInfoFromID, getRoleParams } from "../../../db/utils" import { CURRENTLY_SUPPORTED_LEVELS, getBasePermissions, } from "../../../utilities/security" import sdk from "../../../sdk" import { isV2 } from "../views" +import { removeFromArray } from "../../../utilities" type ResourcePermissions = Record< string, { role: string; type: PermissionSource } > +export const enum PermissionUpdateType { + REMOVE = "remove", + ADD = "add", +} + export async function getInheritablePermissions( resourceId: string ): Promise { @@ -100,3 +108,89 @@ export async function getDependantResources( return } + +export async function updatePermissionOnRole( + { + roleId, + resourceId, + level, + }: { roleId: string; resourceId: string; level: PermissionLevel }, + updateType: PermissionUpdateType +) { + const db = context.getAppDB() + const remove = updateType === PermissionUpdateType.REMOVE + const isABuiltin = roles.isBuiltin(roleId) + const dbRoleId = roles.getDBRoleID(roleId) + const dbRoles = await getAllDBRoles(db) + const docUpdates: Role[] = [] + + // the permission is for a built in, make sure it exists + if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) { + const builtin = roles.getBuiltinRoles()[roleId] + builtin._id = roles.getDBRoleID(builtin._id!) + dbRoles.push(builtin) + } + + // now try to find any roles which need updated, e.g. removing the + // resource from another role and then adding to the new role + for (let role of dbRoles) { + let updated = false + const rolePermissions: Record = role.permissions + ? role.permissions + : {} + // make sure its an array, also handle migrating + if ( + !rolePermissions[resourceId] || + !Array.isArray(rolePermissions[resourceId]) + ) { + rolePermissions[resourceId] = + typeof rolePermissions[resourceId] === "string" + ? [rolePermissions[resourceId] as unknown as PermissionLevel] + : [] + } + // handle the removal/updating the role which has this permission first + // the updating (role._id !== dbRoleId) is required because a resource/level can + // only be permitted in a single role (this reduces hierarchy confusion and simplifies + // the general UI for this, rather than needing to show everywhere it is used) + if ( + (role._id !== dbRoleId || remove) && + rolePermissions[resourceId].indexOf(level) !== -1 + ) { + removeFromArray(rolePermissions[resourceId], level) + updated = true + } + // handle the adding, we're on the correct role, at it to this + if (!remove && role._id === dbRoleId) { + const set = new Set(rolePermissions[resourceId]) + rolePermissions[resourceId] = [...set.add(level)] + updated = true + } + // handle the update, add it to bulk docs to perform at end + if (updated) { + role.permissions = rolePermissions + docUpdates.push(role) + } + } + + const response = await db.bulkDocs(docUpdates) + return response.map(resp => { + const version = docUpdates.find(role => role._id === resp.id)?.version + const _id = roles.getExternalRoleID(resp.id, version) + return { + _id, + rev: resp.rev, + error: resp.error, + reason: resp.reason, + } + }) +} + +// utility function to stop this repetition - permissions always stored under roles +export async function getAllDBRoles(db: Database) { + const body = await db.allDocs( + getRoleParams(null, { + include_docs: true, + }) + ) + return body.rows.map(row => row.doc!) +} diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index cd0fdf077e..24e4da3172 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,5 +1,6 @@ import { FieldType, + PermissionLevel, RelationSchemaField, RenameColumn, Table, @@ -9,7 +10,7 @@ import { ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" -import { context, docIds, HTTPError } from "@budibase/backend-core" +import { context, docIds, HTTPError, roles } from "@budibase/backend-core" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -22,6 +23,7 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" +import { PermissionUpdateType, updatePermissionOnRole } from "../permissions" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -191,7 +193,30 @@ export async function create( ): Promise { await guardViewSchema(tableId, viewRequest) - return pickApi(tableId).create(tableId, viewRequest) + const view = await pickApi(tableId).create(tableId, viewRequest) + + // Set permissions to be the same as the table + const tablePerms = await sdk.permissions.getResourcePerms(tableId) + const readRole = tablePerms[PermissionLevel.READ]?.role + const writeRole = tablePerms[PermissionLevel.WRITE]?.role + await updatePermissionOnRole( + { + roleId: readRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.READ, + }, + PermissionUpdateType.ADD + ) + await updatePermissionOnRole( + { + roleId: writeRole || roles.BUILTIN_ROLE_IDS.BASIC, + resourceId: view.id, + level: PermissionLevel.WRITE, + }, + PermissionUpdateType.ADD + ) + + return view } export async function update(tableId: string, view: ViewV2): Promise { From 01415fb06673e3834d464d35226820ea22789352 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 15:43:58 +0100 Subject: [PATCH 047/105] Quick fix. --- packages/shared-core/src/filters.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 45e9a7c6d0..ef0500b01a 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -124,7 +124,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters[logical]) { + if (filters?.[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) From f6649b294b1b62e54009a280d1ccaa0bdcb6b6ce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 30 Sep 2024 15:54:01 +0100 Subject: [PATCH 048/105] Remove an implicit any from removeInvalidFilters. --- packages/server/src/sdk/app/rows/queryUtils.ts | 13 +++++++++---- packages/types/src/sdk/search.ts | 10 ++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 7ef776a989..c4f4a1eb2c 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -16,11 +16,11 @@ export const removeInvalidFilters = ( validFields = validFields.map(f => f.toLowerCase()) for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { - const filter = result[filterKey] - if (!filter || typeof filter !== "object") { - continue - } if (isLogicalSearchOperator(filterKey)) { + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } const resultingConditions: SearchFilters[] = [] for (const condition of filter.conditions) { const resultingCondition = removeInvalidFilters(condition, validFields) @@ -36,6 +36,11 @@ export const removeInvalidFilters = ( continue } + const filter = result[filterKey] + if (!filter || typeof filter !== "object") { + continue + } + for (const columnKey of Object.keys(filter)) { const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map( c => c.toLowerCase() diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 1d5b36031c..647a9e7d00 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -68,6 +68,8 @@ type RangeFilter = Record< [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never } +type LogicalFilter = { conditions: SearchFilters[] } + export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter export interface SearchFilters { @@ -92,12 +94,8 @@ export interface SearchFilters { // specific document type (such as just rows) documentType?: DocumentType - [LogicalOperator.AND]?: { - conditions: SearchFilters[] - } - [LogicalOperator.OR]?: { - conditions: SearchFilters[] - } + [LogicalOperator.AND]?: LogicalFilter + [LogicalOperator.OR]?: LogicalFilter } export type SearchFilterKey = keyof Omit< From 8462b4e839dde7f138f97a5361ca9eb45510eda1 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 30 Sep 2024 16:10:39 +0000 Subject: [PATCH 049/105] Bump version to 2.32.9 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 10d36c9eaf..5a279b3e44 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.8", + "version": "2.32.9", "npmClient": "yarn", "packages": [ "packages/*", From 6e660151bdef6cd5c9130e09a695bc47b80d4b0c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 18:06:47 +0100 Subject: [PATCH 050/105] backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals. --- .../server/src/api/controllers/row/views.ts | 48 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/shared-core/src/filters.ts | 208 +++++++++++++++++- packages/shared-core/src/utils.ts | 110 ++++++++- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 16 +- packages/types/src/documents/app/view.ts | 8 +- packages/types/src/sdk/search.ts | 5 + 8 files changed, 372 insertions(+), 32 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 68958da8e7..398121f49b 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -5,6 +5,9 @@ import { SearchViewRowRequest, SearchFilterKey, LogicalOperator, + RequiredKeys, + RowSearchParams, + LegacyFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -17,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -25,23 +28,35 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query = dataFilters.buildQueryLegacy(view.query) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -49,15 +64,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -65,25 +81,29 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) - const result = await sdk.rows.search({ - viewId: view.id, + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { tableId: view.tableId, + viewId: view.id, query: enrichedQuery, + fields: viewFields, ...getSortOptions(body, view), limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, - }) + } + const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ef0500b01a..18ce4b6ed7 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -19,9 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -124,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -304,10 +307,143 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -export const buildQuery = (filter: SearchFilter[]) => { + +const buildCondition = (expression: LegacyFilter) => { + // Filter body + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.toLocaleString().startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + +export const buildQueryLegacy = ( + filter?: LegacyFilter[] | SearchFilters +): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + +export const buildQuery = ( + filter?: SearchFilterGroup | LegacyFilter[] +): SearchFilters | undefined => { + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) + + if (!parsedFilter) { + return + } + + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters + ?.map(x => buildCondition(x)) + .filter(filter => filter), + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 81fab659c6..b441791751 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,20 @@ -import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types" +import { + LegacyFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" + +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type WhitelistedFilters = [ + keyof LegacyFilter, + LegacyFilter[keyof LegacyFilter] +][] export function unreachable( value: never, @@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: WhitelistedFilters = + filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const value = filter[key] + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, []) + + const migratedFilter: LegacyFilter = Object.fromEntries( + whiteListedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,7 +1,11 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -9,3 +13,11 @@ export type SearchFilter = { value: any externalType?: string } + +// this is a type purely used by the UI +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: LegacyFilter[] +} diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index a957564039..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 647a9e7d00..d41bb0fb99 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -191,6 +191,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", From f31c7c3487d2b2cd91454f1e89a9be12d2ddae7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 10:55:25 +0200 Subject: [PATCH 051/105] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aab846e704..09273abdce 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1738,6 +1738,40 @@ describe.each([ }) }) + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + it("views without data can be returned", async () => { const response = await config.api.viewV2.search(view.id) expect(response.rows).toHaveLength(0) From 975e348de5b9bed62030ed73b7db00cef700df0d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 10:25:15 +0100 Subject: [PATCH 052/105] Check options.fields are in the table. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..eb04e9fe62 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -89,7 +89,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) From 4b65ce4f8b51e55c9dc33de08337c8b8e0cbd7dd Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 1 Oct 2024 09:31:57 +0000 Subject: [PATCH 053/105] Bump version to 2.32.10 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5a279b3e44..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.9", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", From 119767a30e970220a64f7aa8a43c25cc349c56b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 12:20:18 +0200 Subject: [PATCH 054/105] Cleanup --- packages/server/src/api/controllers/row/views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index b8d01424f2..622688deb6 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -14,7 +14,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view: ViewV2 = await sdk.views.get(viewId) + const view = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } From 522941abf004cae1e04f650a0128df73aca2814a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:31:41 +0100 Subject: [PATCH 055/105] PR comments. --- packages/shared-core/src/filters.ts | 9 +-------- packages/shared-core/src/utils.ts | 19 +++++++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 18ce4b6ed7..b10375acb0 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -599,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -613,16 +610,12 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { - if (!filters) { - return filters - } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b441791751..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -11,10 +11,7 @@ import { removeKeyNumbering } from "./filters" // an array of keys from filter type to properties that are in the type // this can then be converted using .fromEntries to an object -type WhitelistedFilters = [ - keyof LegacyFilter, - LegacyFilter[keyof LegacyFilter] -][] +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] export function unreachable( value: never, @@ -141,7 +138,7 @@ export const processSearchFilters = ( groups: [], } - const filterWhitelistKeys = [ + const filterAllowedKeys = [ "field", "operator", "value", @@ -181,10 +178,10 @@ export const processSearchFilters = ( return acc } - const whiteListedFilterSettings: WhitelistedFilters = - filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { const value = filter[key] - if (filterWhitelistKeys.includes(key)) { + if (filterAllowedKeys.includes(key)) { if (key === "field") { acc.push([key, removeKeyNumbering(value)]) } else { @@ -192,10 +189,12 @@ export const processSearchFilters = ( } } return acc - }, []) + }, + [] + ) const migratedFilter: LegacyFilter = Object.fromEntries( - whiteListedFilterSettings + allowedFilterSettings ) as LegacyFilter baseGroup.filters!.push(migratedFilter) From 19407d5e37bc4a2e937c4e153a8bdf74fd46a83f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:38:02 +0100 Subject: [PATCH 056/105] Check filters have been provided. --- packages/server/src/sdk/app/rows/search/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } From d7873c5c6e5cd6908f4fc75bc44a53b9af6436aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:16 +0100 Subject: [PATCH 057/105] Test fix. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- packages/server/src/sdk/app/rows/search.ts | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..092a851e14 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], + // ["in-memory", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 87129fdbc8..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -81,12 +81,13 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { // Delete extraneous search params that cannot be overridden delete options.query.onEmptyFilter - options = searchInputMapping(table, options) - const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as From 4d33106b450781573083fd6359f6c1a3297e374e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:44 +0100 Subject: [PATCH 058/105] Undo commenting out other DBs. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 092a851e14..1ec5ca792a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], + ["in-memory", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" From 987a24fabc85d76bb3332e364d877a4349083692 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 11:48:14 +0100 Subject: [PATCH 059/105] wip --- packages/backend-core/src/sql/sql.ts | 87 ++++++++++++------- .../src/api/routes/tests/viewV2.spec.ts | 39 +++++++-- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b7bf5bc102..54ca5a0135 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -71,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) { }) } -function getTableName(table?: Table): string | undefined { - // SQS uses the table ID rather than the table name - if ( - table?.sourceType === TableSourceType.INTERNAL || - table?.sourceId === INTERNAL_TABLE_SOURCE_ID - ) { - return table?._id - } else { - return table?.name - } -} - function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { if (Array.isArray(query)) { return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery) @@ -180,15 +168,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] @@ -813,17 +799,39 @@ class InternalBuilder { return query } + getTableName(t?: Table | string): string { + let table: Table + if (typeof t === "string") { + if (!this.query.meta.tables?.[t]) { + throw new Error(`Table ${t} not found`) + } + table = this.query.meta.tables[t] + } else if (t) { + table = t + } else { + table = this.table + } + + let name = table.name + if ( + (table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID) && + table._id + ) { + // SQS uses the table ID rather than the table name + name = table._id + } + const aliases = this.query.tableAliases || {} + return aliases[name] ? aliases[name] : name + } + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { - const primary = this.table.primary - const aliases = this.query.tableAliases - const aliased = - this.table.name && aliases?.[this.table.name] - ? aliases[this.table.name] - : this.table.name - if (!primary) { + if (!this.table.primary) { throw new Error("SQL counting requires primary key to be supplied") } - return query.countDistinct(`${aliased}.${primary[0]} as total`) + return query.countDistinct( + `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + ) } addAggregations( @@ -831,8 +839,9 @@ class InternalBuilder { aggregations: Aggregation[] ): Knex.QueryBuilder { const fields = this.query.resource?.fields || [] + const tableName = this.getTableName() if (fields.length > 0) { - query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + query = query.groupBy(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType @@ -861,10 +870,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort, resource } = this.query const primaryKey = this.table.primary - const tableName = getTableName(this.table) - const aliases = this.query.tableAliases - const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name + const aliased = this.getTableName() if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -1508,18 +1514,35 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } + private getTableName( + table: Table, + aliases?: Record + ): string | undefined { + let name = table.name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + if (!table._id) { + return + } + // SQS uses the table ID rather than the table name + name = table._id + } + return aliases?.[name] ? aliases[name] : name + } + convertJsonStringColumns>( table: Table, results: T[], aliases?: Record ): T[] { - const tableName = getTableName(table) + const tableName = this.getTableName(table, aliases) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = (tableName && aliases?.[tableName]) || tableName - const fullName = `${aliasedTableName}.${name}` + const fullName = `${tableName}.${name}` for (let row of results) { if (typeof row[fullName as keyof T] === "string") { row[fullName as keyof T] = JSON.parse(row[fullName]) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 09273abdce..0f4e6c961c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - ["lucene", undefined], - ["sqs", undefined], + // ["lucene", undefined], + // ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2458,6 +2458,33 @@ describe.each([ expect("_id" in row).toBe(false) } }) + + it.only("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + quantity: { + visible: true, + field: "quantity", + }, + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + for (const row of response.rows) { + expect(row["quantity"]).toBeGreaterThan(0) + expect(row["Total Price"]).toBeGreaterThan(0) + } + }) }) }) From 84f7a477a1a39953ce11bc3aeedce5ec0e872a34 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 12:00:03 +0100 Subject: [PATCH 060/105] Fix the binding drawer for default values. --- .../DataTable/modals/CreateEditColumn.svelte | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..a1bd54715b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -21,6 +21,7 @@ PROTECTED_EXTERNAL_COLUMNS, canHaveDefaultColumn, } from "@budibase/shared-core" + import { makePropSafe } from "@budibase/string-templates" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" @@ -46,6 +47,7 @@ import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" import { isEnabled } from "helpers/featureFlags" + import { getUserBindings } from "dataBinding" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -191,6 +193,19 @@ fieldId: makeFieldId(t.type, t.subtype), ...t, })) + $: defaultValueBindings = [ + { + type: "context", + runtimeBinding: `${makePropSafe("now")}`, + readableBinding: `Date`, + category: "Date", + icon: "Date", + display: { + name: "Server date", + }, + }, + ...getUserBindings(), + ] const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -781,9 +796,8 @@ setRequired(false) } }} - bindings={getBindings({ table })} + bindings={defaultValueBindings} allowJS - context={rowGoldenSample} />
{/if} From 97b70e1f5a0282f7dd997c78e91e57eb8fe24103 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 13:21:04 +0200 Subject: [PATCH 061/105] Change tableid for source id --- .../src/api/routes/tests/search.spec.ts | 161 +++++++++--------- 1 file changed, 80 insertions(+), 81 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..ac189464a3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -64,7 +64,7 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let client: Knex | undefined - let table: Table + let sourceId: string let rows: Row[] async function basicRelationshipTables(type: RelationshipType) { @@ -74,7 +74,7 @@ describe.each([ }, generator.guid().substring(0, 10) ) - table = await createTable( + sourceId = await createTable( { name: { name: "name", type: FieldType.STRING }, //@ts-ignore - API accepts this structure, will build out rest of definition @@ -83,7 +83,7 @@ describe.each([ relationshipType: type, name: "productCat", fieldName: "product", - tableId: relatedTable._id!, + tableId: relatedTable, constraints: { type: "array", }, @@ -92,8 +92,7 @@ describe.each([ generator.guid().substring(0, 10) ) return { - relatedTable: await config.api.table.get(relatedTable._id!), - table, + relatedTable: await config.api.table.get(relatedTable), } } @@ -137,17 +136,18 @@ describe.each([ }) async function createTable(schema: TableSchema, name?: string) { - return await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema, name }) ) + return table._id! } async function createRows(arr: Record[]) { // Shuffling to avoid false positives given a fixed order - await config.api.row.bulkImport(table._id!, { + await config.api.row.bulkImport(sourceId, { rows: _.shuffle(arr), }) - rows = await config.api.row.fetch(table._id!) + rows = await config.api.row.fetch(sourceId) } class SearchAssertion { @@ -332,7 +332,7 @@ describe.each([ } function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: table._id! }) + return new SearchAssertion({ ...query, tableId: sourceId }) } function expectQuery(query: SearchFilters) { @@ -341,7 +341,7 @@ describe.each([ describe("boolean", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, }) await createRows([{ isTrue: true }, { isTrue: false }]) @@ -482,7 +482,7 @@ describe.each([ }) ) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, appointment: { name: "appointment", type: FieldType.DATETIME }, single_user: { @@ -764,7 +764,7 @@ describe.each([ describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, }) await createRows([{ name: "foo" }, { name: "bar" }]) @@ -1055,7 +1055,7 @@ describe.each([ datasourceId: datasource!._id!, }) - table = resp.datasource.entities![tableName] + sourceId = resp.datasource.entities![tableName]._id! await createRows([{ name: "foo" }, { name: "bar" }]) }) @@ -1079,7 +1079,7 @@ describe.each([ describe("numbers", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, }) await createRows([{ age: 1 }, { age: 10 }]) @@ -1252,7 +1252,7 @@ describe.each([ const JAN_10TH = "2020-01-10T00:00:00.000Z" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ dob: { name: "dob", type: FieldType.DATETIME }, }) @@ -1399,7 +1399,7 @@ describe.each([ const NULL_TIME__ID = `null_time__id` beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ timeid: { name: "timeid", type: FieldType.STRING }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, }) @@ -1560,7 +1560,7 @@ describe.each([ describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ numbers: { name: "numbers", type: FieldType.ARRAY, @@ -1657,7 +1657,7 @@ describe.each([ let BIG = "9223372036854775807" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ num: { name: "num", type: FieldType.BIGINT }, }) await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) @@ -1762,7 +1762,7 @@ describe.each([ isInternal && describe("auto", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ auto: { name: "auto", type: FieldType.AUTO, @@ -1912,21 +1912,18 @@ describe.each([ // be stable or pagination will break. We don't want the user to need // to specify an order for pagination to work. it("is stable without a sort specified", async () => { - let { rows: fullRowList } = await config.api.row.search( - table._id!, - { - tableId: table._id!, - query: {}, - } - ) + let { rows: fullRowList } = await config.api.row.search(sourceId, { + tableId: sourceId, + query: {}, + }) // repeat the search many times to check the first row is always the same let bookmark: string | number | undefined, hasNextPage: boolean | undefined = true, rowCount: number = 0 do { - const response = await config.api.row.search(table._id!, { - tableId: table._id!, + const response = await config.api.row.search(sourceId, { + tableId: sourceId, limit: 1, paginate: true, query: {}, @@ -1949,8 +1946,8 @@ describe.each([ // eslint-disable-next-line no-constant-condition while (true) { - const response = await config.api.row.search(table._id!, { - tableId: table._id!, + const response = await config.api.row.search(sourceId, { + tableId: sourceId, limit: 3, query: {}, bookmark, @@ -1973,7 +1970,7 @@ describe.each([ describe("field name 1:name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ "1:name": { name: "1:name", type: FieldType.STRING }, }) await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) @@ -2007,14 +2004,14 @@ describe.each([ }, "array" ) - table = await createTable( + sourceId = await createTable( { relationship: { type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_ONE, name: "relationship", fieldName: "relate", - tableId: arrayTable._id!, + tableId: arrayTable, constraints: { type: "array", }, @@ -2030,17 +2027,17 @@ describe.each([ "main" ) const arrayRows = await Promise.all([ - config.api.row.save(arrayTable._id!, { + config.api.row.save(arrayTable, { name: "foo", array: ["option 1"], }), - config.api.row.save(arrayTable._id!, { + config.api.row.save(arrayTable, { name: "bar", array: ["option 2"], }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { relationship: [arrayRows[0]._id, arrayRows[1]._id], }), ]) @@ -2059,7 +2056,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - table = await createTable({ + sourceId = await createTable({ user: { name: "user", type: FieldType.BB_REFERENCE_SINGLE, @@ -2139,7 +2136,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - table = await createTable({ + sourceId = await createTable({ users: { name: "users", type: FieldType.BB_REFERENCE, @@ -2260,15 +2257,15 @@ describe.each([ ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "foo", productCat: [productCatRows[0]._id], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "bar", productCat: [productCatRows[1]._id], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "baz", productCat: [], }), @@ -2304,7 +2301,7 @@ describe.each([ const { relatedTable } = await basicRelationshipTables( RelationshipType.MANY_TO_ONE ) - const mainRow = await config.api.row.save(table._id!, { + const mainRow = await config.api.row.save(sourceId, { name: "foo", }) for (let i = 0; i < 11; i++) { @@ -2329,7 +2326,7 @@ describe.each([ }) ;(isSqs || isLucene) && describe("relations to same table", () => { - let relatedTable: Table, relatedRows: Row[] + let relatedTable: string, relatedRows: Row[] beforeAll(async () => { relatedTable = await createTable( @@ -2338,36 +2335,36 @@ describe.each([ }, "productCategory" ) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, name: "related1", fieldName: "main1", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, related2: { type: FieldType.LINK, name: "related2", fieldName: "main2", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, }) relatedRows = await Promise.all([ - config.api.row.save(relatedTable._id!, { name: "foo" }), - config.api.row.save(relatedTable._id!, { name: "bar" }), - config.api.row.save(relatedTable._id!, { name: "baz" }), - config.api.row.save(relatedTable._id!, { name: "boo" }), + config.api.row.save(relatedTable, { name: "foo" }), + config.api.row.save(relatedTable, { name: "bar" }), + config.api.row.save(relatedTable, { name: "baz" }), + config.api.row.save(relatedTable, { name: "boo" }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test", related1: [relatedRows[0]._id!], related2: [relatedRows[1]._id!], }), - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test2", related1: [relatedRows[2]._id!], related2: [relatedRows[3]._id!], @@ -2430,7 +2427,7 @@ describe.each([ isInternal && describe("no column error backwards compat", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2453,7 +2450,7 @@ describe.each([ !isLucene && describe("row counting", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2488,7 +2485,7 @@ describe.each([ describe("Invalid column definitions", () => { beforeAll(async () => { // need to create an invalid table - means ignoring typescript - table = await createTable({ + sourceId = await createTable({ // @ts-ignore invalid: { type: FieldType.STRING, @@ -2518,7 +2515,7 @@ describe.each([ "special (%s) case", column => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ [column]: { name: column, type: FieldType.STRING, @@ -2543,8 +2540,8 @@ describe.each([ describe("sample data", () => { beforeAll(async () => { await config.api.application.addSampleData(config.appId!) - table = DEFAULT_EMPLOYEE_TABLE_SCHEMA - rows = await config.api.row.fetch(table._id!) + sourceId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id! + rows = await config.api.row.fetch(sourceId) }) it("should be able to search sample data", async () => { @@ -2567,7 +2564,7 @@ describe.each([ const earlyDate = "2024-07-03T10:00:00.000Z", laterDate = "2024-07-03T11:00:00.000Z" beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ date: { name: "date", type: FieldType.DATETIME, @@ -2610,7 +2607,7 @@ describe.each([ "ชื่อผู้ใช้", // Thai for "username" ])("non-ascii column name: %s", name => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ [name]: { name, type: FieldType.STRING, @@ -2637,7 +2634,7 @@ describe.each([ isInternal && describe("space at end of column name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ "name ": { name: "name ", type: FieldType.STRING, @@ -2672,7 +2669,7 @@ describe.each([ ;(isSqs || isInMemory) && describe("space at start of column name", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ " name": { name: " name", type: FieldType.STRING, @@ -2705,7 +2702,7 @@ describe.each([ isSqs && describe("duplicate columns", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2713,7 +2710,7 @@ describe.each([ }) await context.doInAppContext(config.getAppId(), async () => { const db = context.getAppDB() - const tableDoc = await db.get
(table._id!) + const tableDoc = await db.get
(sourceId) tableDoc.schema.Name = { name: "Name", type: FieldType.STRING, @@ -2747,7 +2744,7 @@ describe.each([ type: FieldType.STRING, }, }) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING, @@ -2756,15 +2753,15 @@ describe.each([ name: "rel", type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_MANY, - tableId: toRelateTable._id!, + tableId: toRelateTable, fieldName: "rel", }, }) const [row1, row2] = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "tag 1" }), - config.api.row.save(toRelateTable._id!, { name: "tag 2" }), + config.api.row.save(toRelateTable, { name: "tag 1" }), + config.api.row.save(toRelateTable, { name: "tag 2" }), ]) - row = await config.api.row.save(table._id!, { + row = await config.api.row.save(sourceId, { name: "product 1", rel: [row1._id, row2._id], }) @@ -2783,7 +2780,7 @@ describe.each([ !isInternal && describe("search by composite key", () => { beforeAll(async () => { - table = await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema: { idColumn1: { @@ -2798,6 +2795,7 @@ describe.each([ primary: ["idColumn1", "idColumn2"], }) ) + sourceId = table._id! await createRows([{ idColumn1: 1, idColumn2: 2 }]) }) @@ -2819,13 +2817,13 @@ describe.each([ isSql && describe("primaryDisplay", () => { beforeAll(async () => { - let toRelateTable = await createTable({ + let toRelateTableId = await createTable({ name: { name: "name", type: FieldType.STRING, }, }) - table = await config.api.table.save( + const table = await config.api.table.save( tableForDatasource(datasource, { schema: { name: { @@ -2836,13 +2834,14 @@ describe.each([ name: "link", type: FieldType.LINK, relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTable._id!, + tableId: toRelateTableId, fieldName: "link", }, }, }) ) - toRelateTable = await config.api.table.get(toRelateTable._id!) + sourceId = table._id! + const toRelateTable = await config.api.table.get(toRelateTableId) await config.api.table.save({ ...toRelateTable, primaryDisplay: "link", @@ -2851,7 +2850,7 @@ describe.each([ config.api.row.save(toRelateTable._id!, { name: "test" }), ]) await Promise.all([ - config.api.row.save(table._id!, { + config.api.row.save(sourceId, { name: "test", link: relatedRows.map(row => row._id), }), @@ -2870,7 +2869,7 @@ describe.each([ !isLucene && describe("$and", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -2999,7 +2998,7 @@ describe.each([ !isLucene && describe("$or", () => { beforeAll(async () => { - table = await createTable({ + sourceId = await createTable({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -3159,20 +3158,20 @@ describe.each([ row[name] = i } const relatedTable = await createTable(relatedSchema) - table = await createTable({ + sourceId = await createTable({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, name: "related1", fieldName: "main1", - tableId: relatedTable._id!, + tableId: relatedTable, relationshipType: RelationshipType.MANY_TO_MANY, }, }) relatedRows = await Promise.all([ - config.api.row.save(relatedTable._id!, row), + config.api.row.save(relatedTable, row), ]) - await config.api.row.save(table._id!, { + await config.api.row.save(sourceId, { name: "foo", related1: [relatedRows[0]._id], }) From b88e63d490923a6c11a000d58bdfe733dc3c5ced Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 13:24:38 +0200 Subject: [PATCH 062/105] Helpers not changing state --- packages/server/src/api/routes/tests/search.spec.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ac189464a3..09ae252115 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -74,7 +74,7 @@ describe.each([ }, generator.guid().substring(0, 10) ) - sourceId = await createTable( + const tableId = await createTable( { name: { name: "name", type: FieldType.STRING }, //@ts-ignore - API accepts this structure, will build out rest of definition @@ -93,6 +93,7 @@ describe.each([ ) return { relatedTable: await config.api.table.get(relatedTable), + tableId, } } @@ -2246,9 +2247,10 @@ describe.each([ let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - const { relatedTable } = await basicRelationshipTables( + const { relatedTable, tableId } = await basicRelationshipTables( RelationshipType.ONE_TO_MANY ) + sourceId = tableId productCategoryTable = relatedTable productCatRows = await Promise.all([ @@ -2298,9 +2300,10 @@ describe.each([ isSql && describe("big relations", () => { beforeAll(async () => { - const { relatedTable } = await basicRelationshipTables( + const { relatedTable, tableId } = await basicRelationshipTables( RelationshipType.MANY_TO_ONE ) + sourceId = tableId const mainRow = await config.api.row.save(sourceId, { name: "foo", }) From f00593ff26ca42e2e1521048aab079b29a3742d3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 1 Oct 2024 12:25:41 +0100 Subject: [PATCH 063/105] pr comments --- .../tests/scenarios/branching.spec.ts | 10 +++++----- .../tests/utilities/AutomationTestBuilder.ts | 20 +++++++++---------- .../server/src/definitions/automations.ts | 4 ++-- packages/server/src/threads/automation.ts | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts index 76e04afdd3..032b729e44 100644 --- a/packages/server/src/automations/tests/scenarios/branching.spec.ts +++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts @@ -30,7 +30,7 @@ describe("Branching automations", () => { .appAction({ fields: {} }) .serverLog( { text: "Starting automation" }, - { stepName: "FirstLog", id: firstLogId } + { stepName: "FirstLog", stepId: firstLogId } ) .branch({ topLevelBranch1: { @@ -38,14 +38,14 @@ describe("Branching automations", () => { stepBuilder .serverLog( { text: "Branch 1" }, - { id: "66666666-6666-6666-6666-666666666666" } + { stepId: "66666666-6666-6666-6666-666666666666" } ) .branch({ branch1: { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.1" }, - { id: branch1LogId } + { stepId: branch1LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: true }, @@ -55,7 +55,7 @@ describe("Branching automations", () => { steps: stepBuilder => stepBuilder.serverLog( { text: "Branch 1.2" }, - { id: branch2LogId } + { stepId: branch2LogId } ), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, @@ -68,7 +68,7 @@ describe("Branching automations", () => { }, topLevelBranch2: { steps: stepBuilder => - stepBuilder.serverLog({ text: "Branch 2" }, { id: branch2Id }), + stepBuilder.serverLog({ text: "Branch 2" }, { stepId: branch2Id }), condition: { equal: { [`{{ steps.${firstLogId}.success }}`]: false }, }, diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 6af18cd27e..6aaf22cd6a 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -64,9 +64,9 @@ class BaseStepBuilder { stepId: TStep, stepSchema: Omit, inputs: AutomationStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { - const id = opts?.id || uuidv4() + const id = opts?.stepId || uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, @@ -107,7 +107,7 @@ class BaseStepBuilder { // STEPS createRow( inputs: CreateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.CREATE_ROW, @@ -119,7 +119,7 @@ class BaseStepBuilder { updateRow( inputs: UpdateRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.UPDATE_ROW, @@ -131,7 +131,7 @@ class BaseStepBuilder { deleteRow( inputs: DeleteRowStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.DELETE_ROW, @@ -143,7 +143,7 @@ class BaseStepBuilder { sendSmtpEmail( inputs: SmtpEmailStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, @@ -155,7 +155,7 @@ class BaseStepBuilder { executeQuery( inputs: ExecuteQueryStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, @@ -167,7 +167,7 @@ class BaseStepBuilder { queryRows( inputs: QueryRowsStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.QUERY_ROWS, @@ -178,7 +178,7 @@ class BaseStepBuilder { } loop( inputs: LoopStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.LOOP, @@ -190,7 +190,7 @@ class BaseStepBuilder { serverLog( input: ServerLogStepInputs, - opts?: { stepName?: string; id?: string } + opts?: { stepName?: string; stepId?: string } ): this { return this.step( AutomationActionStepId.SERVER_LOG, diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 44758b727b..9433075da7 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,8 +15,8 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] - stepsById?: Record - stepsByName?: Record + stepsById: Record + stepsByName: Record env?: Record trigger: any } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 7788744ea2..3b47634663 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -74,7 +74,7 @@ class Orchestrator { private job: Job private loopStepOutputs: LoopStep[] private stopped: boolean - private executionOutput: AutomationContext + private executionOutput: Omit constructor(job: AutomationJob) { let automation = job.data.automation @@ -458,9 +458,9 @@ class Orchestrator { inputs: steps[stepToLoopIndex].inputs, }) - this.context.stepsById![steps[stepToLoopIndex].id] = tempOutput + this.context.stepsById[steps[stepToLoopIndex].id] = tempOutput const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id - this.context.stepsByName![stepName] = tempOutput + this.context.stepsByName[stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") From ae4f7ae4b45ab4e30778eddff0938017924a22d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:04:01 +0100 Subject: [PATCH 064/105] Implement group by and add a test for it. --- packages/backend-core/src/sql/sql.ts | 19 +++++++++------ .../src/api/routes/tests/viewV2.spec.ts | 23 +++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 54ca5a0135..14e32623e3 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -799,6 +799,14 @@ class InternalBuilder { return query } + isSqs(t?: Table): boolean { + const table = t || this.table + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) + } + getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { @@ -813,11 +821,7 @@ class InternalBuilder { } let name = table.name - if ( - (table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID) && - table._id - ) { + if (this.isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } @@ -830,7 +834,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName(this.table)}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as total` ) } @@ -842,10 +846,11 @@ class InternalBuilder { const tableName = this.getTableName() if (fields.length > 0) { query = query.groupBy(fields.map(field => `${tableName}.${field}`)) + query = query.select(fields.map(field => `${tableName}.${field}`)) } for (const aggregation of aggregations) { const op = aggregation.calculationType - const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + const field = `${tableName}.${aggregation.field} as ${aggregation.name}` switch (op) { case CalculationType.COUNT: query = query.count(field) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 0f4e6c961c..b03c445b78 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -39,13 +39,13 @@ import { } from "@budibase/backend-core" describe.each([ - // ["lucene", undefined], - // ["sqs", undefined], + ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2459,7 +2459,7 @@ describe.each([ } }) - it.only("should be able to group by a basic field", async () => { + it("should be able to group by a basic field", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -2480,9 +2480,14 @@ describe.each([ query: {}, }) + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + for (const row of response.rows) { - expect(row["quantity"]).toBeGreaterThan(0) - expect(row["Total Price"]).toBeGreaterThan(0) + expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) } }) }) From addd54a8e8baab79811e6f71aab728f32239577b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 15:39:33 +0100 Subject: [PATCH 065/105] Fix generic-sql.spec.ts --- packages/backend-core/src/sql/sql.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 14e32623e3..0f72eea96e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -811,7 +811,10 @@ class InternalBuilder { let table: Table if (typeof t === "string") { if (!this.query.meta.tables?.[t]) { - throw new Error(`Table ${t} not found`) + // This can legitimately happen in custom queries, where the user is + // querying against a table that may not have been imported into + // Budibase. + return t } table = this.query.meta.tables[t] } else if (t) { @@ -1547,12 +1550,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { if (!this._isJsonColumn(field)) { continue } - const fullName = `${tableName}.${name}` + const fullName = `${tableName}.${name}` as keyof T for (let row of results) { - if (typeof row[fullName as keyof T] === "string") { - row[fullName as keyof T] = JSON.parse(row[fullName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } - if (typeof row[name as keyof T] === "string") { + if (typeof row[name] === "string") { row[name as keyof T] = JSON.parse(row[name]) } } From 7cee1509aa3ee47677e3116121beab3b57a6deef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:17:11 +0100 Subject: [PATCH 066/105] Fix sqlAlias.spec.ts --- packages/backend-core/src/sql/sql.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0f72eea96e..105116e828 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -810,13 +810,18 @@ class InternalBuilder { getTableName(t?: Table | string): string { let table: Table if (typeof t === "string") { - if (!this.query.meta.tables?.[t]) { + if (this.query.table?.name === t) { + table = this.query.table + } else if (this.query.meta.table?.name === t) { + table = this.query.meta.table + } else if (!this.query.meta.tables?.[t]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return t + } else { + table = this.query.meta.tables[t] } - table = this.query.meta.tables[t] } else if (t) { table = t } else { From 4165c6cab42affee7f40f5493e29e06a90c67ec4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:06:40 +0100 Subject: [PATCH 067/105] Test all aggregation types. --- .../src/api/routes/tests/viewV2.spec.ts | 55 +++++++++++++++++++ packages/server/src/integrations/postgres.ts | 3 +- .../src/utilities/rowProcessor/index.ts | 19 +++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index b03c445b78..1d6c1d50cd 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2490,6 +2490,61 @@ describe.each([ expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) } }) + + it.each([ + CalculationType.COUNT, + CalculationType.SUM, + CalculationType.AVG, + CalculationType.MIN, + CalculationType.MAX, + ])("should be able to calculate $type", async type => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + aggregate: { + visible: true, + calculationType: type, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + function calculate( + type: CalculationType, + numbers: number[] + ): number { + switch (type) { + case CalculationType.COUNT: + return numbers.length + case CalculationType.SUM: + return numbers.reduce((a, b) => a + b, 0) + case CalculationType.AVG: + return numbers.reduce((a, b) => a + b, 0) / numbers.length + case CalculationType.MIN: + return Math.min(...numbers) + case CalculationType.MAX: + return Math.max(...numbers) + } + } + + const prices = rows.map(row => row.price) + const expected = calculate(type, prices) + const actual = response.rows[0].aggregate + + if (type === CalculationType.AVG) { + // The average calculation can introduce floating point rounding + // errors, so we need to compare to within a small margin of + // error. + expect(actual).toBeCloseTo(expected) + } else { + expect(actual).toEqual(expected) + } + }) }) }) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 3652864991..ce8b21eede 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -272,7 +272,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { try { const bindings = query.bindings || [] this.log(query.sql, bindings) - return await client.query(query.sql, bindings) + const result = await client.query(query.sql, bindings) + return result } catch (err: any) { await this.closeConnection() let readableMessage = getReadableErrorMessage( diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index bddd590f25..7332f8b244 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -426,6 +426,25 @@ export async function coreOutputProcessing( } } } + + if (sdk.views.isView(source)) { + const calculationFields = Object.keys( + helpers.views.calculationFields(source) + ) + + // We ensure all calculation fields are returned as numbers. During the + // testing of this feature it was discovered that the COUNT operation + // returns a string for MySQL, MariaDB, and Postgres. But given that all + // calculation fields should be numbers, we blanket make sure of that + // here. + for (const key of calculationFields) { + for (const row of rows) { + if (typeof row[key] === "string") { + row[key] = parseFloat(row[key]) + } + } + } + } } if (!isUserMetadataTable(table._id!)) { From 1dea53f5976aeee63c3fe5f5b8087ac64a42fb90 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 1 Oct 2024 16:25:48 +0100 Subject: [PATCH 068/105] Refresh data when adding columns --- .../DataTable/modals/grid/GridCreateColumnModal.svelte | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte index 2040f66706..d031e752cd 100644 --- a/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte @@ -2,7 +2,12 @@ import { getContext } from "svelte" import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte" - const { datasource } = getContext("grid") + const { datasource, rows } = getContext("grid") + + const onUpdate = async () => { + await datasource.actions.refreshDefinition() + await rows.actions.refreshData() + } - + From 13248c409f358cb3c0a791bce59b8e05c01247da Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 16:44:16 +0100 Subject: [PATCH 069/105] Respond to PR comment. --- packages/server/src/integrations/postgres.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index ce8b21eede..3652864991 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -272,8 +272,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { try { const bindings = query.bindings || [] this.log(query.sql, bindings) - const result = await client.query(query.sql, bindings) - return result + return await client.query(query.sql, bindings) } catch (err: any) { await this.closeConnection() let readableMessage = getReadableErrorMessage( From c4a6a92bdbbfee4768c5a77b10aef3c5c8986a43 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:03:06 +0100 Subject: [PATCH 070/105] PR comments --- packages/frontend-core/src/api/ai.js | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/api/ai.js b/packages/frontend-core/src/api/ai.js index 702ce87cdb..7fa756a19e 100644 --- a/packages/frontend-core/src/api/ai.js +++ b/packages/frontend-core/src/api/ai.js @@ -4,7 +4,7 @@ export const buildAIEndpoints = API => ({ */ generateCronExpression: async ({ prompt }) => { return await API.post({ - url: "/api/ai/generate/cron", + url: "/api/ai/cron", body: { prompt }, }) }, diff --git a/packages/pro b/packages/pro index dcc9e50b80..f35190a594 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit dcc9e50b8064a2097d408771462ad80f48de7ff6 +Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 From a28a64f9d8b1053509f58db6acc6ee0c89d92109 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 17:05:38 +0100 Subject: [PATCH 071/105] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f35190a594..aca9828117 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f35190a594afb04525d0bc4405bea8a04bd62b42 +Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7 From 77856eb35a8072c8c71d63178c6cc8a13ef3de0b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 1 Oct 2024 17:23:21 +0100 Subject: [PATCH 072/105] Add a test to make sure fields on the underlying table that are required are not required on the view. --- .../src/api/routes/tests/viewV2.spec.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..f76b6eb470 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2546,6 +2546,51 @@ describe.each([ } }) }) + + !isLucene && + it("should not need required fields to be present", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + await Promise.all([ + config.api.row.save(table._id!, { name: "Steve", age: 30 }), + config.api.row.save(table._id!, { name: "Jane", age: 31 }), + ]) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].sum).toEqual(61) + }) }) describe("permissions", () => { From 08f1c4dadc668db7fe645f01336832341546828e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:35:15 +0100 Subject: [PATCH 073/105] Update packages/backend-core/src/sql/sql.ts Co-authored-by: Adria Navarro --- packages/backend-core/src/sql/sql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 105116e828..e55524a67e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1542,7 +1542,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // SQS uses the table ID rather than the table name name = table._id } - return aliases?.[name] ? aliases[name] : name + return aliases?.[name] || name } convertJsonStringColumns>( From cc6b2f6717cce5ea7dfb2830aa1f1e4de9410e68 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:35:47 +0100 Subject: [PATCH 074/105] add failing test --- .../src/api/routes/tests/viewV2.spec.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index f76b6eb470..954047d536 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -25,6 +25,7 @@ import { ViewFieldMetadata, FeatureFlag, BBReferenceFieldSubType, + ViewCalculationFieldMetadata, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -540,6 +541,31 @@ describe.each([ status: 201, }) }) + + it.only("can create a view with calculation fields", async () => { + let view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + }, + }) + + let sum = view.schema!.sum as ViewCalculationFieldMetadata + expect(sum).toBeDefined() + expect(sum.calculationType).toEqual(CalculationType.SUM) + expect(sum.field).toEqual("Price") + + view = await config.api.viewV2.get(view.id) + sum = view.schema!.sum as ViewCalculationFieldMetadata + expect(sum).toBeDefined() + expect(sum.calculationType).toEqual(CalculationType.SUM) + expect(sum.field).toEqual("Price") + }) }) describe("update", () => { From ddd229062c20e37f2860c8a07583688f67c0ed1c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:39:54 +0100 Subject: [PATCH 075/105] Rename total field when doing row counts. --- packages/backend-core/src/sql/sql.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e55524a67e..701797329f 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -842,7 +842,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName()}.${this.table.primary[0]} as total` + `${this.getTableName()}.${this.table.primary[0]} as __bb_total` ) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d5c0560d9b..3d6bf39d3f 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -57,8 +57,8 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && "total" in response[0]) { - const total = response[0].total + if (response && response.length === 1 && "__bb_total" in response[0]) { + const total = response[0].__bb_total return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From 7b9af81fd510f2aa87c757173abd028c068cebba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:44:20 +0100 Subject: [PATCH 076/105] Clean up params and isSqs --- packages/backend-core/src/sql/sql.ts | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 701797329f..627be039ca 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -87,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +function isSqs(table: Table): boolean { + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -799,37 +806,34 @@ class InternalBuilder { return query } - isSqs(t?: Table): boolean { - const table = t || this.table - return ( - table.sourceType === TableSourceType.INTERNAL || - table.sourceId === INTERNAL_TABLE_SOURCE_ID - ) + isSqs(): boolean { + return isSqs(this.table) } - getTableName(t?: Table | string): string { + getTableName(tableOrName?: Table | string): string { let table: Table - if (typeof t === "string") { - if (this.query.table?.name === t) { + if (typeof tableOrName === "string") { + const name = tableOrName + if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === t) { + } else if (this.query.meta.table?.name === name) { table = this.query.meta.table - } else if (!this.query.meta.tables?.[t]) { + } else if (!this.query.meta.tables?.[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. - return t + return name } else { - table = this.query.meta.tables[t] + table = this.query.meta.tables[name] } - } else if (t) { - table = t + } else if (tableOrName) { + table = tableOrName } else { table = this.table } let name = table.name - if (this.isSqs(table) && table._id) { + if (isSqs(table) && table._id) { // SQS uses the table ID rather than the table name name = table._id } From 4dd6afd4352ba774bde40764cf6d9a315d983d7b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 09:57:18 +0100 Subject: [PATCH 077/105] Symbolise the special __bb_total count field name. --- packages/backend-core/src/sql/sql.ts | 4 +++- packages/server/src/sdk/app/rows/utils.ts | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 627be039ca..3585dacbed 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -43,6 +43,8 @@ import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any +export const COUNT_FIELD_NAME = "__bb_total" + function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) @@ -846,7 +848,7 @@ class InternalBuilder { throw new Error("SQL counting requires primary key to be supplied") } return query.countDistinct( - `${this.getTableName()}.${this.table.primary[0]} as __bb_total` + `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}` ) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 3d6bf39d3f..e1b5615046 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" +import { COUNT_FIELD_NAME } from "@budibase/backend-core/src/sql/sql" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -57,8 +58,8 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && "__bb_total" in response[0]) { - const total = response[0].__bb_total + if (response && response.length === 1 && COUNT_FIELD_NAME in response[0]) { + const total = response[0][COUNT_FIELD_NAME] return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From ee897e4d7ef6afbeb56bb4068c9e1e4ba44959f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 10:05:56 +0100 Subject: [PATCH 078/105] Fix imports. --- packages/backend-core/src/sql/index.ts | 2 +- packages/server/src/sdk/app/rows/utils.ts | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/index.ts b/packages/backend-core/src/sql/index.ts index 16b718d2e6..816b3d60a5 100644 --- a/packages/backend-core/src/sql/index.ts +++ b/packages/backend-core/src/sql/index.ts @@ -1,5 +1,5 @@ export * as utils from "./utils" -export { default as Sql } from "./sql" +export { default as Sql, COUNT_FIELD_NAME } from "./sql" export { default as SqlTable } from "./sqlTable" export * as designDoc from "./designDoc" diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index e1b5615046..6ef4dcbc8e 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -20,9 +20,8 @@ import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" -import { docIds } from "@budibase/backend-core" +import { docIds, sql } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" -import { COUNT_FIELD_NAME } from "@budibase/backend-core/src/sql/sql" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -58,8 +57,12 @@ export function getSQLClient(datasource: Datasource): SqlClient { export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { - if (response && response.length === 1 && COUNT_FIELD_NAME in response[0]) { - const total = response[0][COUNT_FIELD_NAME] + if ( + response && + response.length === 1 && + sql.COUNT_FIELD_NAME in response[0] + ) { + const total = response[0][sql.COUNT_FIELD_NAME] return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") From 0679ec89931879158f2ef3fd43bc9de89540321b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 10:36:45 +0100 Subject: [PATCH 079/105] Make sure calculation views are created and returned correctly. --- .../src/api/routes/tests/viewV2.spec.ts | 4 ++- packages/server/src/sdk/app/views/index.ts | 31 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 954047d536..669d35ba5b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -542,7 +542,7 @@ describe.each([ }) }) - it.only("can create a view with calculation fields", async () => { + it("can create a view with calculation fields", async () => { let view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -555,6 +555,8 @@ describe.each([ }, }) + expect(Object.keys(view.schema!)).toHaveLength(1) + let sum = view.schema!.sum as ViewCalculationFieldMetadata expect(sum).toBeDefined() expect(sum.calculationType).toEqual(CalculationType.SUM) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 24e4da3172..d218a3c7e8 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -258,19 +258,12 @@ export async function enrichSchema( view: ViewV2, tableSchema: TableSchema ): Promise { - const tableCache: Record = {} - async function populateRelTableSchema( tableId: string, viewFields: Record ) { - if (!tableCache[tableId]) { - tableCache[tableId] = await sdk.tables.getTable(tableId) - } - const relTable = tableCache[tableId] - + const relTable = await sdk.tables.getTable(tableId) const result: Record = {} - for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) { @@ -299,15 +292,22 @@ export async function enrichSchema( const viewSchema = view.schema || {} const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null) - for (const key of Object.keys(tableSchema).filter( - k => tableSchema[k].visible !== false - )) { + + const visibleSchemaFields = Object.keys(viewSchema).filter( + key => viewSchema[key].visible !== false + ) + const visibleTableFields = Object.keys(tableSchema).filter( + key => tableSchema[key].visible !== false + ) + const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields]) + + for (const key of visibleFields) { // if nothing specified in view, then it is not visible const ui = viewSchema[key] || { visible: false } schema[key] = { - ...tableSchema[key], + ...(tableSchema[key] || {}), ...ui, - order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order, + order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order, columns: undefined, } @@ -319,10 +319,7 @@ export async function enrichSchema( } } - return { - ...view, - schema: schema, - } + return { ...view, schema } } export function syncSchema( From e08c3b8574956da13f9261d731910986f75a9f3f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 12:26:03 +0200 Subject: [PATCH 080/105] Run view tests --- .../src/api/routes/tests/search.spec.ts | 5502 +++++++++-------- .../server/src/tests/utilities/api/row.ts | 4 +- 2 files changed, 2772 insertions(+), 2734 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 09ae252115..86ae0acf9b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -28,6 +28,7 @@ import { RowSearchParams, SearchFilters, SearchResponse, + SearchRowRequest, SortOrder, SortType, Table, @@ -64,7 +65,7 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let client: Knex | undefined - let sourceId: string + let tableOrViewId: string let rows: Row[] async function basicRelationshipTables(type: RelationshipType) { @@ -145,395 +146,402 @@ describe.each([ async function createRows(arr: Record[]) { // Shuffling to avoid false positives given a fixed order - await config.api.row.bulkImport(sourceId, { - rows: _.shuffle(arr), - }) - rows = await config.api.row.fetch(sourceId) + for (const row of _.shuffle(arr)) { + await config.api.row.save(tableOrViewId, row) + } + rows = await config.api.row.fetch(tableOrViewId) } - class SearchAssertion { - constructor(private readonly query: RowSearchParams) {} + describe.each([ + ["table", createTable], + // [ + // "view", + // async (schema: TableSchema, name?: string) => { + // const tableId = await createTable(schema, name) + // const view = await config.api.viewV2.create({ + // tableId: tableId, + // name: generator.guid(), + // schema: Object.keys(schema).reduce>( + // (viewSchema, fieldName) => { + // const field = schema[fieldName] + // viewSchema[fieldName] = { + // visible: field.visible ?? true, + // readonly: false, + // } + // return viewSchema + // }, + // {} + // ), + // }) + // return view.id + // }, + // ], + ])("from %s", (__, createSource) => { + class SearchAssertion { + constructor(private readonly query: SearchRowRequest) {} - private async performSearch(): Promise> { - if (isInMemory) { - return dataFilters.search(_.cloneDeep(rows), this.query) - } else { - const sourceId = this.query.viewId || this.query.tableId - if (!sourceId) { - throw new Error("No source ID provided") - } - return config.api.row.search(sourceId, this.query) - } - } - - // We originally used _.isMatch to compare rows, but found that when - // comparing arrays it would return true if the source array was a subset of - // the target array. This would sometimes create false matches. This - // function is a more strict version of _.isMatch that only returns true if - // the source array is an exact match of the target. - // - // _.isMatch("100", "1") also returns true which is not what we want. - private isMatch>(expected: T, found: T) { - if (!expected) { - throw new Error("Expected is undefined") - } - if (!found) { - return false - } - - for (const key of Object.keys(expected)) { - if (Array.isArray(expected[key])) { - if (!Array.isArray(found[key])) { - return false - } - if (expected[key].length !== found[key].length) { - return false - } - if (!_.isMatch(found[key], expected[key])) { - return false - } - } else if (typeof expected[key] === "object") { - if (!this.isMatch(expected[key], found[key])) { - return false - } + private async performSearch(): Promise> { + if (isInMemory) { + return dataFilters.search(_.cloneDeep(rows), { + ...this.query, + tableId: tableOrViewId, + }) } else { - if (expected[key] !== found[key]) { - return false + return config.api.row.search(tableOrViewId, this.query) + } + } + + // We originally used _.isMatch to compare rows, but found that when + // comparing arrays it would return true if the source array was a subset of + // the target array. This would sometimes create false matches. This + // function is a more strict version of _.isMatch that only returns true if + // the source array is an exact match of the target. + // + // _.isMatch("100", "1") also returns true which is not what we want. + private isMatch>(expected: T, found: T) { + if (!expected) { + throw new Error("Expected is undefined") + } + if (!found) { + return false + } + + for (const key of Object.keys(expected)) { + if (Array.isArray(expected[key])) { + if (!Array.isArray(found[key])) { + return false + } + if (expected[key].length !== found[key].length) { + return false + } + if (!_.isMatch(found[key], expected[key])) { + return false + } + } else if (typeof expected[key] === "object") { + if (!this.isMatch(expected[key], found[key])) { + return false + } + } else { + if (expected[key] !== found[key]) { + return false + } } } - } - return true - } - - // This function exists to ensure that the same row is not matched twice. - // When a row gets matched, we make sure to remove it from the list of rows - // we're matching against. - private popRow( - expectedRow: T, - foundRows: T[] - ): NonNullable { - const row = foundRows.find(row => this.isMatch(expectedRow, row)) - if (!row) { - const fields = Object.keys(expectedRow) - // To make the error message more readable, we only include the fields - // that are present in the expected row. - const searchedObjects = foundRows.map(row => _.pick(row, fields)) - throw new Error( - `Failed to find row:\n\n${JSON.stringify( - expectedRow, - null, - 2 - )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}` - ) + return true } - foundRows.splice(foundRows.indexOf(row), 1) - return row - } - - // Asserts that the query returns rows matching exactly the set of rows - // passed in. The order of the rows matters. Rows returned in an order - // different to the one passed in will cause the assertion to fail. Extra - // rows returned by the query will also cause the assertion to fail. - async toMatchExactly(expectedRows: any[]) { - const response = await this.performSearch() - const cloned = cloneDeep(response) - const foundRows = response.rows - - // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toHaveLength(expectedRows.length) - // eslint-disable-next-line jest/no-standalone-expect - expect([...foundRows]).toEqual( - expectedRows.map((expectedRow: any) => - expect.objectContaining(this.popRow(expectedRow, foundRows)) - ) - ) - return cloned - } - - // Asserts that the query returns rows matching exactly the set of rows - // passed in. The order of the rows is not important, but extra rows will - // cause the assertion to fail. - async toContainExactly(expectedRows: any[]) { - const response = await this.performSearch() - const cloned = cloneDeep(response) - const foundRows = response.rows - - // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toHaveLength(expectedRows.length) - // eslint-disable-next-line jest/no-standalone-expect - expect([...foundRows]).toEqual( - expect.arrayContaining( - expectedRows.map((expectedRow: any) => - expect.objectContaining(this.popRow(expectedRow, foundRows)) + // This function exists to ensure that the same row is not matched twice. + // When a row gets matched, we make sure to remove it from the list of rows + // we're matching against. + private popRow( + expectedRow: T, + foundRows: T[] + ): NonNullable { + const row = foundRows.find(row => this.isMatch(expectedRow, row)) + if (!row) { + const fields = Object.keys(expectedRow) + // To make the error message more readable, we only include the fields + // that are present in the expected row. + const searchedObjects = foundRows.map(row => _.pick(row, fields)) + throw new Error( + `Failed to find row:\n\n${JSON.stringify( + expectedRow, + null, + 2 + )}\n\nin\n\n${JSON.stringify(searchedObjects, null, 2)}` ) - ) - ) - return cloned - } - - // Asserts that the query returns some property values - this cannot be used - // to check row values, however this shouldn't be important for checking properties - // typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...) - async toMatch(properties: Record) { - const response = await this.performSearch() - const cloned = cloneDeep(response) - const keys = Object.keys(properties) as Array> - for (let key of keys) { - // eslint-disable-next-line jest/no-standalone-expect - expect(response[key]).toBeDefined() - if (properties[key]) { - // eslint-disable-next-line jest/no-standalone-expect - expect(response[key]).toEqual(properties[key]) } - } - return cloned - } - // Asserts that the query doesn't return a property, e.g. pagination parameters. - async toNotHaveProperty(properties: (keyof SearchResponse)[]) { - const response = await this.performSearch() - const cloned = cloneDeep(response) - for (let property of properties) { + foundRows.splice(foundRows.indexOf(row), 1) + return row + } + + // Asserts that the query returns rows matching exactly the set of rows + // passed in. The order of the rows matters. Rows returned in an order + // different to the one passed in will cause the assertion to fail. Extra + // rows returned by the query will also cause the assertion to fail. + async toMatchExactly(expectedRows: any[]) { + const response = await this.performSearch() + const cloned = cloneDeep(response) + const foundRows = response.rows + // eslint-disable-next-line jest/no-standalone-expect - expect(response[property]).toBeUndefined() - } - return cloned - } - - // Asserts that the query returns rows matching the set of rows passed in. - // The order of the rows is not important. Extra rows will not cause the - // assertion to fail. - async toContain(expectedRows: any[]) { - const response = await this.performSearch() - const cloned = cloneDeep(response) - const foundRows = response.rows - - // eslint-disable-next-line jest/no-standalone-expect - expect([...foundRows]).toEqual( - expect.arrayContaining( + expect(foundRows).toHaveLength(expectedRows.length) + // eslint-disable-next-line jest/no-standalone-expect + expect([...foundRows]).toEqual( expectedRows.map((expectedRow: any) => expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) - ) - return cloned - } - - async toFindNothing() { - await this.toContainExactly([]) - } - - async toHaveLength(length: number) { - const { rows: foundRows } = await this.performSearch() - - // eslint-disable-next-line jest/no-standalone-expect - expect(foundRows).toHaveLength(length) - } - } - - function expectSearch(query: Omit) { - return new SearchAssertion({ ...query, tableId: sourceId }) - } - - function expectQuery(query: SearchFilters) { - return expectSearch({ query }) - } - - describe("boolean", () => { - beforeAll(async () => { - sourceId = await createTable({ - isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, - }) - await createRows([{ isTrue: true }, { isTrue: false }]) - }) - - describe("equal", () => { - it("successfully finds true row", async () => { - await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ - { isTrue: true }, - ]) - }) - - it("successfully finds false row", async () => { - await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ - { isTrue: false }, - ]) - }) - }) - - describe("notEqual", () => { - it("successfully finds false row", async () => { - await expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ - { isTrue: false }, - ]) - }) - - it("successfully finds true row", async () => { - await expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ - { isTrue: true }, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds true row", async () => { - await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ - { isTrue: true }, - ]) - }) - - it("successfully finds false row", async () => { - await expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ - { isTrue: false }, - ]) - }) - }) - - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "isTrue", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ isTrue: false }, { isTrue: true }]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "isTrue", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ isTrue: true }, { isTrue: false }]) - }) - }) - }) - - !isInMemory && - describe("bindings", () => { - let globalUsers: any = [] - - const serverTime = new Date() - - // In MariaDB and MySQL we only store dates to second precision, so we need - // to remove milliseconds from the server time to ensure searches work as - // expected. - serverTime.setMilliseconds(0) - - const future = new Date(serverTime.getTime() + 1000 * 60 * 60 * 24 * 30) - - const rows = (currentUser: User) => { - return [ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - { name: currentUser.firstName, appointment: future.toISOString() }, - { name: "serverDate", appointment: serverTime.toISOString() }, - { - name: "single user, session user", - single_user: JSON.stringify(currentUser), - }, - { - name: "single user", - single_user: JSON.stringify(globalUsers[0]), - }, - { - name: "deprecated single user, session user", - deprecated_single_user: JSON.stringify([currentUser]), - }, - { - name: "deprecated single user", - deprecated_single_user: JSON.stringify([globalUsers[0]]), - }, - { - name: "multi user", - multi_user: JSON.stringify(globalUsers), - }, - { - name: "multi user with session user", - multi_user: JSON.stringify([...globalUsers, currentUser]), - }, - { - name: "deprecated multi user", - deprecated_multi_user: JSON.stringify(globalUsers), - }, - { - name: "deprecated multi user with session user", - deprecated_multi_user: JSON.stringify([ - ...globalUsers, - currentUser, - ]), - }, - ] + return cloned } - beforeAll(async () => { - // Set up some global users - globalUsers = await Promise.all( - Array(2) - .fill(0) - .map(async () => { - const globalUser = await config.globalUser() - const userMedataId = globalUser._id - ? dbCore.generateUserMetadataID(globalUser._id) - : null - return { - _id: globalUser._id, - _meta: userMedataId, - } - }) + // Asserts that the query returns rows matching exactly the set of rows + // passed in. The order of the rows is not important, but extra rows will + // cause the assertion to fail. + async toContainExactly(expectedRows: any[]) { + const response = await this.performSearch() + const cloned = cloneDeep(response) + const foundRows = response.rows + + // eslint-disable-next-line jest/no-standalone-expect + expect(foundRows).toHaveLength(expectedRows.length) + // eslint-disable-next-line jest/no-standalone-expect + expect([...foundRows]).toEqual( + expect.arrayContaining( + expectedRows.map((expectedRow: any) => + expect.objectContaining(this.popRow(expectedRow, foundRows)) + ) + ) ) + return cloned + } - sourceId = await createTable({ - name: { name: "name", type: FieldType.STRING }, - appointment: { name: "appointment", type: FieldType.DATETIME }, - single_user: { - name: "single_user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - deprecated_single_user: { - name: "deprecated_single_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - }, - multi_user: { - name: "multi_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - constraints: { - type: "array", - }, - }, - deprecated_multi_user: { - name: "deprecated_multi_user", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USERS, - constraints: { - type: "array", - }, - }, + // Asserts that the query returns some property values - this cannot be used + // to check row values, however this shouldn't be important for checking properties + // typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...) + async toMatch(properties: Record) { + const response = await this.performSearch() + const cloned = cloneDeep(response) + const keys = Object.keys(properties) as Array> + for (let key of keys) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[key]).toBeDefined() + if (properties[key]) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[key]).toEqual(properties[key]) + } + } + return cloned + } + + // Asserts that the query doesn't return a property, e.g. pagination parameters. + async toNotHaveProperty(properties: (keyof SearchResponse)[]) { + const response = await this.performSearch() + const cloned = cloneDeep(response) + for (let property of properties) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[property]).toBeUndefined() + } + return cloned + } + + // Asserts that the query returns rows matching the set of rows passed in. + // The order of the rows is not important. Extra rows will not cause the + // assertion to fail. + async toContain(expectedRows: any[]) { + const response = await this.performSearch() + const cloned = cloneDeep(response) + const foundRows = response.rows + + // eslint-disable-next-line jest/no-standalone-expect + expect([...foundRows]).toEqual( + expect.arrayContaining( + expectedRows.map((expectedRow: any) => + expect.objectContaining(this.popRow(expectedRow, foundRows)) + ) + ) + ) + return cloned + } + + async toFindNothing() { + await this.toContainExactly([]) + } + + async toHaveLength(length: number) { + const { rows: foundRows } = await this.performSearch() + + // eslint-disable-next-line jest/no-standalone-expect + expect(foundRows).toHaveLength(length) + } + } + + function expectSearch(query: SearchRowRequest) { + return new SearchAssertion(query) + } + + function expectQuery(query: SearchFilters) { + return expectSearch({ query }) + } + + describe("boolean", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, }) - await createRows(rows(config.getUser())) + await createRows([{ isTrue: true }, { isTrue: false }]) }) - // !! Current User is auto generated per run - it("should return all rows matching the session user firstname", async () => { - await expectQuery({ - equal: { name: "{{ [user].firstName }}" }, - }).toContainExactly([ - { - name: config.getUser().firstName, - appointment: future.toISOString(), - }, - ]) + describe("equal", () => { + it("successfully finds true row", async () => { + await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ + { isTrue: true }, + ]) + }) + + it("successfully finds false row", async () => { + await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ + { isTrue: false }, + ]) + }) }) - !isLucene && - it("should return all rows matching the session user firstname when logical operator used", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { name: "{{ [user].firstName }}" } }], + describe("notEqual", () => { + it("successfully finds false row", async () => { + await expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ + { isTrue: false }, + ]) + }) + + it("successfully finds true row", async () => { + await expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ + { isTrue: true }, + ]) + }) + }) + + describe("oneOf", () => { + it("successfully finds true row", async () => { + await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ + { isTrue: true }, + ]) + }) + + it("successfully finds false row", async () => { + await expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ + { isTrue: false }, + ]) + }) + }) + + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "isTrue", + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ isTrue: false }, { isTrue: true }]) + }) + + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "isTrue", + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([{ isTrue: true }, { isTrue: false }]) + }) + }) + }) + + !isInMemory && + describe("bindings", () => { + let globalUsers: any = [] + + const serverTime = new Date() + + // In MariaDB and MySQL we only store dates to second precision, so we need + // to remove milliseconds from the server time to ensure searches work as + // expected. + serverTime.setMilliseconds(0) + + const future = new Date(serverTime.getTime() + 1000 * 60 * 60 * 24 * 30) + + const rows = (currentUser: User) => { + return [ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + { name: currentUser.firstName, appointment: future.toISOString() }, + { name: "serverDate", appointment: serverTime.toISOString() }, + { + name: "single user, session user", + single_user: currentUser, }, + { + name: "single user", + single_user: globalUsers[0], + }, + { + name: "deprecated single user, session user", + deprecated_single_user: [currentUser], + }, + { + name: "deprecated single user", + deprecated_single_user: [globalUsers[0]], + }, + { + name: "multi user", + multi_user: globalUsers, + }, + { + name: "multi user with session user", + multi_user: [...globalUsers, currentUser], + }, + { + name: "deprecated multi user", + deprecated_multi_user: globalUsers, + }, + { + name: "deprecated multi user with session user", + deprecated_multi_user: [...globalUsers, currentUser], + }, + ] + } + + beforeAll(async () => { + // Set up some global users + globalUsers = await Promise.all( + Array(2) + .fill(0) + .map(async () => { + const globalUser = await config.globalUser() + const userMedataId = globalUser._id + ? dbCore.generateUserMetadataID(globalUser._id) + : null + return { + _id: globalUser._id, + _meta: userMedataId, + } + }) + ) + + tableOrViewId = await createSource({ + name: { name: "name", type: FieldType.STRING }, + appointment: { name: "appointment", type: FieldType.DATETIME }, + single_user: { + name: "single_user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + deprecated_single_user: { + name: "deprecated_single_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + }, + multi_user: { + name: "multi_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: "array", + }, + }, + deprecated_multi_user: { + name: "deprecated_multi_user", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USERS, + constraints: { + type: "array", + }, + }, + }) + await createRows(rows(config.getUser())) + }) + + // !! Current User is auto generated per run + it("should return all rows matching the session user firstname", async () => { + await expectQuery({ + equal: { name: "{{ [user].firstName }}" }, }).toContainExactly([ { name: config.getUser().firstName, @@ -542,488 +550,488 @@ describe.each([ ]) }) - it("should parse the date binding and return all rows after the resolved value", async () => { - await tk.withFreeze(serverTime, async () => { + !isLucene && + it("should return all rows matching the session user firstname when logical operator used", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { name: "{{ [user].firstName }}" } }], + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + ]) + }) + + it("should parse the date binding and return all rows after the resolved value", async () => { + await tk.withFreeze(serverTime, async () => { + await expectQuery({ + range: { + appointment: { + low: "{{ [now] }}", + high: "9999-00-00T00:00:00.000Z", + }, + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + { name: "serverDate", appointment: serverTime.toISOString() }, + ]) + }) + }) + + it("should parse the date binding and return all rows before the resolved value", async () => { await expectQuery({ range: { appointment: { - low: "{{ [now] }}", - high: "9999-00-00T00:00:00.000Z", + low: "0000-00-00T00:00:00.000Z", + high: "{{ [now] }}", }, }, }).toContainExactly([ - { - name: config.getUser().firstName, - appointment: future.toISOString(), - }, + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, { name: "serverDate", appointment: serverTime.toISOString() }, ]) }) + + it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => { + const jsBinding = "return snippets.WeeksAgo();" + const encodedBinding = encodeJSBinding(jsBinding) + + await expectQuery({ + range: { + appointment: { + low: "0000-00-00T00:00:00.000Z", + high: encodedBinding, + }, + }, + }).toContainExactly([ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + ]) + }) + + it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { + const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();` + const encodedBinding = encodeJSBinding(jsBinding) + + await expectQuery({ + range: { + appointment: { + low: "0000-00-00T00:00:00.000Z", + high: encodedBinding, + }, + }, + }).toContainExactly([ + { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, + { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, + ]) + }) + + it("should match a single user row by the session user id", async () => { + await expectQuery({ + equal: { single_user: "{{ [user]._id }}" }, + }).toContainExactly([ + { + name: "single user, session user", + single_user: { _id: config.getUser()._id }, + }, + ]) + }) + + it("should match a deprecated single user row by the session user id", async () => { + await expectQuery({ + equal: { deprecated_single_user: "{{ [user]._id }}" }, + }).toContainExactly([ + { + name: "deprecated single user, session user", + deprecated_single_user: [{ _id: config.getUser()._id }], + }, + ]) + }) + + it("should match the session user id in a multi user field", async () => { + const allUsers = [...globalUsers, config.getUser()].map( + (user: any) => { + return { _id: user._id } + } + ) + + await expectQuery({ + contains: { multi_user: ["{{ [user]._id }}"] }, + }).toContainExactly([ + { + name: "multi user with session user", + multi_user: allUsers, + }, + ]) + }) + + it("should match the session user id in a deprecated multi user field", async () => { + const allUsers = [...globalUsers, config.getUser()].map( + (user: any) => { + return { _id: user._id } + } + ) + + await expectQuery({ + contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, + }).toContainExactly([ + { + name: "deprecated multi user with session user", + deprecated_multi_user: allUsers, + }, + ]) + }) + + it("should not match the session user id in a multi user field", async () => { + await expectQuery({ + notContains: { multi_user: ["{{ [user]._id }}"] }, + notEmpty: { multi_user: true }, + }).toContainExactly([ + { + name: "multi user", + multi_user: globalUsers.map((user: any) => { + return { _id: user._id } + }), + }, + ]) + }) + + it("should not match the session user id in a deprecated multi user field", async () => { + await expectQuery({ + notContains: { deprecated_multi_user: ["{{ [user]._id }}"] }, + notEmpty: { deprecated_multi_user: true }, + }).toContainExactly([ + { + name: "deprecated multi user", + deprecated_multi_user: globalUsers.map((user: any) => { + return { _id: user._id } + }), + }, + ]) + }) + + it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => { + await expectQuery({ + oneOf: { + single_user: [ + "{{ default [user]._id '_empty_' }}", + globalUsers[0]._id, + ], + }, + }).toContainExactly([ + { + name: "single user, session user", + single_user: { _id: config.getUser()._id }, + }, + { + name: "single user", + single_user: { _id: globalUsers[0]._id }, + }, + ]) + }) + + it("should match the session user id and a user table row id using helpers, user binding and a static user id. (deprecated single user)", async () => { + await expectQuery({ + oneOf: { + deprecated_single_user: [ + "{{ default [user]._id '_empty_' }}", + globalUsers[0]._id, + ], + }, + }).toContainExactly([ + { + name: "deprecated single user, session user", + deprecated_single_user: [{ _id: config.getUser()._id }], + }, + { + name: "deprecated single user", + deprecated_single_user: [{ _id: globalUsers[0]._id }], + }, + ]) + }) + + it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => { + await expectQuery({ + oneOf: { + single_user: [ + "{{ default [user]._idx '_empty_' }}", + globalUsers[0]._id, + ], + }, + }).toContainExactly([ + { + name: "single user", + single_user: { _id: globalUsers[0]._id }, + }, + ]) + }) + + it("should resolve 'default' helper to '_empty_' when binding resolves to nothing (deprecated single user)", async () => { + await expectQuery({ + oneOf: { + deprecated_single_user: [ + "{{ default [user]._idx '_empty_' }}", + globalUsers[0]._id, + ], + }, + }).toContainExactly([ + { + name: "deprecated single user", + deprecated_single_user: [{ _id: globalUsers[0]._id }], + }, + ]) + }) }) - it("should parse the date binding and return all rows before the resolved value", async () => { - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: "{{ [now] }}", - }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - { name: "serverDate", appointment: serverTime.toISOString() }, - ]) + describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([{ name: "foo" }, { name: "bar" }]) }) - it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => { - const jsBinding = "return snippets.WeeksAgo();" - const encodedBinding = encodeJSBinding(jsBinding) - - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: encodedBinding, - }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - ]) - }) - - it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => { - const jsBinding = `const currentTime = new Date(${Date.now()})\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();` - const encodedBinding = encodeJSBinding(jsBinding) - - await expectQuery({ - range: { - appointment: { - low: "0000-00-00T00:00:00.000Z", - high: encodedBinding, - }, - }, - }).toContainExactly([ - { name: "foo", appointment: "1982-01-05T00:00:00.000Z" }, - { name: "bar", appointment: "1995-05-06T00:00:00.000Z" }, - ]) - }) - - it("should match a single user row by the session user id", async () => { - await expectQuery({ - equal: { single_user: "{{ [user]._id }}" }, - }).toContainExactly([ - { - name: "single user, session user", - single_user: { _id: config.getUser()._id }, - }, - ]) - }) - - it("should match a deprecated single user row by the session user id", async () => { - await expectQuery({ - equal: { deprecated_single_user: "{{ [user]._id }}" }, - }).toContainExactly([ - { - name: "deprecated single user, session user", - deprecated_single_user: [{ _id: config.getUser()._id }], - }, - ]) - }) - - it("should match the session user id in a multi user field", async () => { - const allUsers = [...globalUsers, config.getUser()].map((user: any) => { - return { _id: user._id } + describe("misc", () => { + it("should return all if no query is passed", async () => { + await expectSearch({} as RowSearchParams).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) }) - await expectQuery({ - contains: { multi_user: ["{{ [user]._id }}"] }, - }).toContainExactly([ - { - name: "multi user with session user", - multi_user: allUsers, - }, - ]) - }) - - it("should match the session user id in a deprecated multi user field", async () => { - const allUsers = [...globalUsers, config.getUser()].map((user: any) => { - return { _id: user._id } + it("should return all if empty query is passed", async () => { + await expectQuery({}).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) }) - await expectQuery({ - contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, - }).toContainExactly([ - { - name: "deprecated multi user with session user", - deprecated_multi_user: allUsers, - }, - ]) - }) - - it("should not match the session user id in a multi user field", async () => { - await expectQuery({ - notContains: { multi_user: ["{{ [user]._id }}"] }, - notEmpty: { multi_user: true }, - }).toContainExactly([ - { - name: "multi user", - multi_user: globalUsers.map((user: any) => { - return { _id: user._id } - }), - }, - ]) - }) - - it("should not match the session user id in a deprecated multi user field", async () => { - await expectQuery({ - notContains: { deprecated_multi_user: ["{{ [user]._id }}"] }, - notEmpty: { deprecated_multi_user: true }, - }).toContainExactly([ - { - name: "deprecated multi user", - deprecated_multi_user: globalUsers.map((user: any) => { - return { _id: user._id } - }), - }, - ]) - }) - - it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => { - await expectQuery({ - oneOf: { - single_user: [ - "{{ default [user]._id '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "single user, session user", - single_user: { _id: config.getUser()._id }, - }, - { - name: "single user", - single_user: { _id: globalUsers[0]._id }, - }, - ]) - }) - - it("should match the session user id and a user table row id using helpers, user binding and a static user id. (deprecated single user)", async () => { - await expectQuery({ - oneOf: { - deprecated_single_user: [ - "{{ default [user]._id '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "deprecated single user, session user", - deprecated_single_user: [{ _id: config.getUser()._id }], - }, - { - name: "deprecated single user", - deprecated_single_user: [{ _id: globalUsers[0]._id }], - }, - ]) - }) - - it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => { - await expectQuery({ - oneOf: { - single_user: [ - "{{ default [user]._idx '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "single user", - single_user: { _id: globalUsers[0]._id }, - }, - ]) - }) - - it("should resolve 'default' helper to '_empty_' when binding resolves to nothing (deprecated single user)", async () => { - await expectQuery({ - oneOf: { - deprecated_single_user: [ - "{{ default [user]._idx '_empty_' }}", - globalUsers[0]._id, - ], - }, - }).toContainExactly([ - { - name: "deprecated single user", - deprecated_single_user: [{ _id: globalUsers[0]._id }], - }, - ]) - }) - }) - - describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { - beforeAll(async () => { - sourceId = await createTable({ - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([{ name: "foo" }, { name: "bar" }]) - }) - - describe("misc", () => { - it("should return all if no query is passed", async () => { - await expectSearch({} as RowSearchParams).toContainExactly([ - { name: "foo" }, - { name: "bar" }, - ]) - }) - - it("should return all if empty query is passed", async () => { - await expectQuery({}).toContainExactly([ - { name: "foo" }, - { name: "bar" }, - ]) - }) - - it("should return all if onEmptyFilter is RETURN_ALL", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toFindNothing() - }) - - it("should respect limit", async () => { - await expectSearch({ - limit: 1, - paginate: true, - query: {}, - }).toHaveLength(1) - }) - }) - - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { name: "foo" } }).toContainExactly([ - { name: "foo" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { name: "none" } }).toFindNothing() - }) - - it("works as an or condition", async () => { - await expectQuery({ - allOr: true, - equal: { name: "foo" }, - oneOf: { name: ["bar"] }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("can have multiple values for same column", async () => { - await expectQuery({ - allOr: true, - equal: { "1:name": "foo", "2:name": "bar" }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - }) - - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ - { name: "bar" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ - { name: "foo" }, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ - { name: "foo" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() - }) - - it("can have multiple values for same column", async () => { - await expectQuery({ - oneOf: { - name: ["foo", "bar"], - }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("splits comma separated strings", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - name: "foo,bar", - }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("trims whitespace", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - name: "foo, bar", - }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - oneOf: { name: [] }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - - it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - oneOf: { name: [] }, - }).toContainExactly([]) - }) - }) - - describe("fuzzy", () => { - it("successfully finds a row", async () => { - await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ - { name: "foo" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ fuzzy: { name: "none" } }).toFindNothing() - }) - }) - - describe("string", () => { - it("successfully finds a row", async () => { - await expectQuery({ string: { name: "fo" } }).toContainExactly([ - { name: "foo" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ string: { name: "none" } }).toFindNothing() - }) - - it("is case-insensitive", async () => { - await expectQuery({ string: { name: "FO" } }).toContainExactly([ - { name: "foo" }, - ]) - }) - }) - - describe("range", () => { - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { name: { low: "a", high: "z" } }, - }).toContainExactly([{ name: "bar" }, { name: "foo" }]) - }) - - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { name: { low: "a", high: "c" } }, - }).toContainExactly([{ name: "bar" }]) - }) - - it("successfully finds a row with a low bound", async () => { - await expectQuery({ - range: { name: { low: "f", high: "z" } }, - }).toContainExactly([{ name: "foo" }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { name: { low: "g", high: "h" } }, - }).toFindNothing() - }) - - !isLucene && - it("ignores low if it's an empty object", async () => { + it("should return all if onEmptyFilter is RETURN_ALL", async () => { await expectQuery({ - // @ts-ignore - range: { name: { low: {}, high: "z" } }, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, }).toContainExactly([{ name: "foo" }, { name: "bar" }]) }) - !isLucene && - it("ignores high if it's an empty object", async () => { + it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { await expectQuery({ - // @ts-ignore - range: { name: { low: "a", high: {} } }, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }).toFindNothing() + }) + + it("should respect limit", async () => { + await expectSearch({ + limit: 1, + paginate: true, + query: {}, + }).toHaveLength(1) + }) + }) + + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { name: "foo" } }).toContainExactly([ + { name: "foo" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { name: "none" } }).toFindNothing() + }) + + it("works as an or condition", async () => { + await expectQuery({ + allOr: true, + equal: { name: "foo" }, + oneOf: { name: ["bar"] }, }).toContainExactly([{ name: "foo" }, { name: "bar" }]) }) - }) - describe("empty", () => { - it("finds no empty rows", async () => { - await expectQuery({ empty: { name: null } }).toFindNothing() + it("can have multiple values for same column", async () => { + await expectQuery({ + allOr: true, + equal: { "1:name": "foo", "2:name": "bar" }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) - it("should not be affected by when filter empty behaviour", async () => { - await expectQuery({ - empty: { name: null }, - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toFindNothing() - }) - }) + describe("notEqual", () => { + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ + { name: "bar" }, + ]) + }) - describe("notEmpty", () => { - it("finds all non-empty rows", async () => { - await expectQuery({ notEmpty: { name: null } }).toContainExactly([ - { name: "foo" }, - { name: "bar" }, - ]) + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ + { name: "foo" }, + ]) + }) }) - it("should not be affected by when filter empty behaviour", async () => { - await expectQuery({ - notEmpty: { name: null }, - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) - }) + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ + { name: "foo" }, + ]) + }) - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "name", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() + }) + + it("can have multiple values for same column", async () => { + await expectQuery({ + oneOf: { + name: ["foo", "bar"], + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("splits comma separated strings", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + name: "foo,bar", + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("trims whitespace", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + name: "foo, bar", + }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => { + await expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + oneOf: { name: [] }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { + await expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + oneOf: { name: [] }, + }).toContainExactly([]) + }) }) - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "name", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + describe("fuzzy", () => { + it("successfully finds a row", async () => { + await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ + { name: "foo" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ fuzzy: { name: "none" } }).toFindNothing() + }) }) - describe("sortType STRING", () => { + describe("string", () => { + it("successfully finds a row", async () => { + await expectQuery({ string: { name: "fo" } }).toContainExactly([ + { name: "foo" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ string: { name: "none" } }).toFindNothing() + }) + + it("is case-insensitive", async () => { + await expectQuery({ string: { name: "FO" } }).toContainExactly([ + { name: "foo" }, + ]) + }) + }) + + describe("range", () => { + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { name: { low: "a", high: "z" } }, + }).toContainExactly([{ name: "bar" }, { name: "foo" }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { name: { low: "a", high: "c" } }, + }).toContainExactly([{ name: "bar" }]) + }) + + it("successfully finds a row with a low bound", async () => { + await expectQuery({ + range: { name: { low: "f", high: "z" } }, + }).toContainExactly([{ name: "foo" }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { name: { low: "g", high: "h" } }, + }).toFindNothing() + }) + + !isLucene && + it("ignores low if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: {}, high: "z" } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + + !isLucene && + it("ignores high if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: "a", high: {} } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + }) + + describe("empty", () => { + it("finds no empty rows", async () => { + await expectQuery({ empty: { name: null } }).toFindNothing() + }) + + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ + empty: { name: null }, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + }).toFindNothing() + }) + }) + + describe("notEmpty", () => { + it("finds all non-empty rows", async () => { + await expectQuery({ notEmpty: { name: null } }).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) + }) + + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ + notEmpty: { name: null }, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) + }) + + describe("sort", () => { it("sorts ascending", async () => { await expectSearch({ query: {}, sort: "name", - sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) }) @@ -1032,345 +1040,355 @@ describe.each([ await expectSearch({ query: {}, sort: "name", - sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) }) - }) - !isInternal && - !isInMemory && - // This test was added because we automatically add in a sort by the - // primary key, and we used to do this unconditionally which caused - // problems because it was possible for the primary key to appear twice - // in the resulting SQL ORDER BY clause, resulting in an SQL error. - // We now check first to make sure that the primary key isn't already - // in the sort before adding it. - describe("sort on primary key", () => { - beforeAll(async () => { - const tableName = structures.uuid().substring(0, 10) - await client!.schema.createTable(tableName, t => { - t.string("name").primary() - }) - const resp = await config.api.datasource.fetchSchema({ - datasourceId: datasource!._id!, - }) - - sourceId = resp.datasource.entities![tableName]._id! - - await createRows([{ name: "foo" }, { name: "bar" }]) + describe("sortType STRING", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "name", + sortType: SortType.STRING, + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) }) - it("should be able to sort by a primary key column ascending", async () => - expectSearch({ - query: {}, - sort: "name", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) - - it("should be able to sort by a primary key column descending", async () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", + sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) }) - }) - }) - describe("numbers", () => { - beforeAll(async () => { - sourceId = await createTable({ - age: { name: "age", type: FieldType.NUMBER }, - }) - await createRows([{ age: 1 }, { age: 10 }]) - }) + !isInternal && + !isInMemory && + // This test was added because we automatically add in a sort by the + // primary key, and we used to do this unconditionally which caused + // problems because it was possible for the primary key to appear twice + // in the resulting SQL ORDER BY clause, resulting in an SQL error. + // We now check first to make sure that the primary key isn't already + // in the sort before adding it. + describe("sort on primary key", () => { + beforeAll(async () => { + const tableName = structures.uuid().substring(0, 10) + await client!.schema.createTable(tableName, t => { + t.string("name").primary() + }) + const resp = await config.api.datasource.fetchSchema({ + datasourceId: datasource!._id!, + }) - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) - }) + tableOrViewId = resp.datasource.entities![tableName]._id! - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { age: 2 } }).toFindNothing() + await createRows([{ name: "foo" }, { name: "bar" }]) + }) + + it("should be able to sort by a primary key column ascending", async () => + expectSearch({ + query: {}, + sort: "name", + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + + it("should be able to sort by a primary key column descending", async () => + expectSearch({ + query: {}, + sort: "name", + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }) }) }) - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { age: 1 } }).toContainExactly([ - { age: 10 }, - ]) + describe("numbers", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + age: { name: "age", type: FieldType.NUMBER }, + }) + await createRows([{ age: 1 }, { age: 10 }]) }) - it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ - { age: 1 }, - ]) - }) - }) + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { age: 1 } }).toContainExactly([ + { age: 1 }, + ]) + }) - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { age: [1] } }).toContainExactly([ - { age: 1 }, - ]) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { age: 2 } }).toFindNothing() + }) }) - it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { age: [2] } }).toFindNothing() + describe("notEqual", () => { + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { age: 1 } }).toContainExactly([ + { age: 10 }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ + { age: 1 }, + ]) + }) }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can convert from a string", async () => { + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { age: [1] } }).toContainExactly([ + { age: 1 }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { age: [2] } }).toFindNothing() + }) + + // I couldn't find a way to make this work in Lucene and given that + // we're getting rid of Lucene soon I wasn't inclined to spend time on + // it. + !isLucene && + it("can convert from a string", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1", + }, + }).toContainExactly([{ age: 1 }]) + }) + + // I couldn't find a way to make this work in Lucene and given that + // we're getting rid of Lucene soon I wasn't inclined to spend time on + // it. + !isLucene && + it("can find multiple values for same column", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1,10", + }, + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) + }) + + describe("range", () => { + it("successfully finds a row", async () => { await expectQuery({ - oneOf: { - // @ts-ignore - age: "1", + range: { age: { low: 1, high: 5 } }, + }).toContainExactly([{ age: 1 }]) + }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { age: { low: 1, high: 10 } }, + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { age: { low: 5, high: 10 } }, + }).toContainExactly([{ age: 10 }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { age: { low: 5, high: 9 } }, + }).toFindNothing() + }) + + it("greater than equal to", async () => { + await expectQuery({ + range: { + age: { low: 10, high: Number.MAX_SAFE_INTEGER }, + }, + }).toContainExactly([{ age: 10 }]) + }) + + it("greater than", async () => { + await expectQuery({ + range: { + age: { low: 5, high: Number.MAX_SAFE_INTEGER }, + }, + }).toContainExactly([{ age: 10 }]) + }) + + it("less than equal to", async () => { + await expectQuery({ + range: { + age: { high: 1, low: Number.MIN_SAFE_INTEGER }, }, }).toContainExactly([{ age: 1 }]) }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can find multiple values for same column", async () => { + it("less than", async () => { await expectQuery({ - oneOf: { - // @ts-ignore - age: "1,10", + range: { + age: { high: 5, low: Number.MIN_SAFE_INTEGER }, }, - }).toContainExactly([{ age: 1 }, { age: 10 }]) + }).toContainExactly([{ age: 1 }]) }) - }) - - describe("range", () => { - it("successfully finds a row", async () => { - await expectQuery({ - range: { age: { low: 1, high: 5 } }, - }).toContainExactly([{ age: 1 }]) }) - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { age: { low: 1, high: 10 } }, - }).toContainExactly([{ age: 1 }, { age: 10 }]) + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "age", + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) + + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "age", + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { age: { low: 5, high: 10 } }, - }).toContainExactly([{ age: 10 }]) - }) + describe("sortType NUMBER", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "age", + sortType: SortType.NUMBER, + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("successfully finds no rows", async () => { - await expectQuery({ - range: { age: { low: 5, high: 9 } }, - }).toFindNothing() - }) - - it("greater than equal to", async () => { - await expectQuery({ - range: { - age: { low: 10, high: Number.MAX_SAFE_INTEGER }, - }, - }).toContainExactly([{ age: 10 }]) - }) - - it("greater than", async () => { - await expectQuery({ - range: { - age: { low: 5, high: Number.MAX_SAFE_INTEGER }, - }, - }).toContainExactly([{ age: 10 }]) - }) - - it("less than equal to", async () => { - await expectQuery({ - range: { - age: { high: 1, low: Number.MIN_SAFE_INTEGER }, - }, - }).toContainExactly([{ age: 1 }]) - }) - - it("less than", async () => { - await expectQuery({ - range: { - age: { high: 5, low: Number.MIN_SAFE_INTEGER }, - }, - }).toContainExactly([{ age: 1 }]) + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "age", + sortType: SortType.NUMBER, + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) }) - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "age", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }]) + describe("dates", () => { + const JAN_1ST = "2020-01-01T00:00:00.000Z" + const JAN_2ND = "2020-01-02T00:00:00.000Z" + const JAN_5TH = "2020-01-05T00:00:00.000Z" + const JAN_9TH = "2020-01-09T00:00:00.000Z" + const JAN_10TH = "2020-01-10T00:00:00.000Z" + + beforeAll(async () => { + tableOrViewId = await createSource({ + dob: { name: "dob", type: FieldType.DATETIME }, + }) + + await createRows([{ dob: JAN_1ST }, { dob: JAN_10TH }]) }) - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "age", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }]) - }) - }) + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ + { dob: JAN_1ST }, + ]) + }) - describe("sortType NUMBER", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "age", - sortType: SortType.NUMBER, - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }]) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing() + }) }) - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "age", - sortType: SortType.NUMBER, - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }]) - }) - }) - }) + describe("notEqual", () => { + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ + { dob: JAN_10TH }, + ]) + }) - describe("dates", () => { - const JAN_1ST = "2020-01-01T00:00:00.000Z" - const JAN_2ND = "2020-01-02T00:00:00.000Z" - const JAN_5TH = "2020-01-05T00:00:00.000Z" - const JAN_9TH = "2020-01-09T00:00:00.000Z" - const JAN_10TH = "2020-01-10T00:00:00.000Z" - - beforeAll(async () => { - sourceId = await createTable({ - dob: { name: "dob", type: FieldType.DATETIME }, + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ + { dob: JAN_1ST }, + ]) + }) }) - await createRows([{ dob: JAN_1ST }, { dob: JAN_10TH }]) - }) + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ + { dob: JAN_1ST }, + ]) + }) - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ - { dob: JAN_1ST }, - ]) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() + }) }) - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing() - }) - }) + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { dob: { low: JAN_1ST, high: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_1ST }]) + }) - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ - { dob: JAN_10TH }, - ]) + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { dob: { low: JAN_1ST, high: JAN_10TH } }, + }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { dob: { low: JAN_5TH, high: JAN_10TH } }, + }).toContainExactly([{ dob: JAN_10TH }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { dob: { low: JAN_5TH, high: JAN_9TH } }, + }).toFindNothing() + }) + + it("greater than equal to", async () => { + await expectQuery({ + range: { + dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() }, + }, + }).toContainExactly([{ dob: JAN_10TH }]) + }) + + it("greater than", async () => { + await expectQuery({ + range: { + dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() }, + }, + }).toContainExactly([{ dob: JAN_10TH }]) + }) + + it("less than equal to", async () => { + await expectQuery({ + range: { + dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() }, + }, + }).toContainExactly([{ dob: JAN_1ST }]) + }) + + it("less than", async () => { + await expectQuery({ + range: { + dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() }, + }, + }).toContainExactly([{ dob: JAN_1ST }]) + }) }) - it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ - { dob: JAN_1ST }, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ - { dob: JAN_1ST }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() - }) - }) - - describe("range", () => { - it("successfully finds a row", async () => { - await expectQuery({ - range: { dob: { low: JAN_1ST, high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }]) - }) - - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { dob: { low: JAN_1ST, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) - }) - - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { dob: { low: JAN_5TH, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_10TH }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { dob: { low: JAN_5TH, high: JAN_9TH } }, - }).toFindNothing() - }) - - it("greater than equal to", async () => { - await expectQuery({ - range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } }, - }).toContainExactly([{ dob: JAN_10TH }]) - }) - - it("greater than", async () => { - await expectQuery({ - range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } }, - }).toContainExactly([{ dob: JAN_10TH }]) - }) - - it("less than equal to", async () => { - await expectQuery({ - range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } }, - }).toContainExactly([{ dob: JAN_1ST }]) - }) - - it("less than", async () => { - await expectQuery({ - range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } }, - }).toContainExactly([{ dob: JAN_1ST }]) - }) - }) - - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "dob", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "dob", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) - }) - - describe("sortType STRING", () => { + describe("sort", () => { it("sorts ascending", async () => { await expectSearch({ query: {}, sort: "dob", - sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) }) @@ -1379,157 +1397,143 @@ describe.each([ await expectSearch({ query: {}, sort: "dob", - sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) }) - }) - }) - }) - - !isInternal && - describe("datetime - time only", () => { - const T_1000 = "10:00:00" - const T_1045 = "10:45:00" - const T_1200 = "12:00:00" - const T_1530 = "15:30:00" - const T_0000 = "00:00:00" - - const UNEXISTING_TIME = "10:01:00" - - const NULL_TIME__ID = `null_time__id` - - beforeAll(async () => { - sourceId = await createTable({ - timeid: { name: "timeid", type: FieldType.STRING }, - time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, - }) - - await createRows([ - { timeid: NULL_TIME__ID, time: null }, - { time: T_1000 }, - { time: T_1045 }, - { time: T_1200 }, - { time: T_1530 }, - { time: T_0000 }, - ]) - }) - - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { time: T_1000 } }).toContainExactly([ - { time: "10:00:00" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - equal: { time: UNEXISTING_TIME }, - }).toFindNothing() - }) - }) - - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ - { timeid: NULL_TIME__ID }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - { time: "00:00:00" }, - ]) - }) - - it("return all when requesting non-existing", async () => { - await expectQuery({ - notEqual: { time: UNEXISTING_TIME }, - }).toContainExactly([ - { timeid: NULL_TIME__ID }, - { time: "10:00:00" }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - { time: "00:00:00" }, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ - { time: "10:00:00" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - oneOf: { time: [UNEXISTING_TIME] }, - }).toFindNothing() - }) - }) - - describe("range", () => { - it("successfully finds a row", async () => { - await expectQuery({ - range: { time: { low: T_1045, high: T_1045 } }, - }).toContainExactly([{ time: "10:45:00" }]) - }) - - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { time: { low: T_1045, high: T_1530 } }, - }).toContainExactly([ - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - ]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, - }).toFindNothing() - }) - }) - - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "time", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([ - { timeid: NULL_TIME__ID }, - { time: "00:00:00" }, - { time: "10:00:00" }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - ]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "time", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([ - { time: "15:30:00" }, - { time: "12:00:00" }, - { time: "10:45:00" }, - { time: "10:00:00" }, - { time: "00:00:00" }, - { timeid: NULL_TIME__ID }, - ]) - }) describe("sortType STRING", () => { it("sorts ascending", async () => { await expectSearch({ query: {}, - sort: "time", + sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) + + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "dob", + sortType: SortType.STRING, + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) + }) + }) + }) + + !isInternal && + describe("datetime - time only", () => { + const T_1000 = "10:00:00" + const T_1045 = "10:45:00" + const T_1200 = "12:00:00" + const T_1530 = "15:30:00" + const T_0000 = "00:00:00" + + const UNEXISTING_TIME = "10:01:00" + + const NULL_TIME__ID = `null_time__id` + + beforeAll(async () => { + tableOrViewId = await createSource({ + timeid: { name: "timeid", type: FieldType.STRING }, + time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, + }) + + await createRows([ + { timeid: NULL_TIME__ID, time: null }, + { time: T_1000 }, + { time: T_1045 }, + { time: T_1200 }, + { time: T_1530 }, + { time: T_0000 }, + ]) + }) + + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + { time: "10:00:00" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + equal: { time: UNEXISTING_TIME }, + }).toFindNothing() + }) + }) + + describe("notEqual", () => { + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + { timeid: NULL_TIME__ID }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ]) + }) + + it("return all when requesting non-existing", async () => { + await expectQuery({ + notEqual: { time: UNEXISTING_TIME }, + }).toContainExactly([ + { timeid: NULL_TIME__ID }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ]) + }) + }) + + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ + { time: "10:00:00" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + oneOf: { time: [UNEXISTING_TIME] }, + }).toFindNothing() + }) + }) + + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { time: { low: T_1045, high: T_1045 } }, + }).toContainExactly([{ time: "10:45:00" }]) + }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { time: { low: T_1045, high: T_1530 } }, + }).toContainExactly([ + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + ]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, + }).toFindNothing() + }) + }) + + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "time", + sortOrder: SortOrder.ASCENDING, }).toMatchExactly([ { timeid: NULL_TIME__ID }, { time: "00:00:00" }, @@ -1544,7 +1548,6 @@ describe.each([ await expectSearch({ query: {}, sort: "time", - sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, }).toMatchExactly([ { time: "15:30:00" }, @@ -1555,1638 +1558,1673 @@ describe.each([ { timeid: NULL_TIME__ID }, ]) }) + + describe("sortType STRING", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "time", + sortType: SortType.STRING, + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([ + { timeid: NULL_TIME__ID }, + { time: "00:00:00" }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + ]) + }) + + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "time", + sortType: SortType.STRING, + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([ + { time: "15:30:00" }, + { time: "12:00:00" }, + { time: "10:45:00" }, + { time: "10:00:00" }, + { time: "00:00:00" }, + { timeid: NULL_TIME__ID }, + ]) + }) + }) }) }) - }) - describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { - beforeAll(async () => { - sourceId = await createTable({ - numbers: { - name: "numbers", - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - inclusion: ["one", "two", "three"], + describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + numbers: { + name: "numbers", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["one", "two", "three"], + }, }, - }, - }) - await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }]) - }) - - describe("contains", () => { - it("successfully finds a row", async () => { - await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ - { numbers: ["one", "two"] }, - ]) + }) + await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }]) }) - it("fails to find nonexistent row", async () => { - await expectQuery({ contains: { numbers: ["none"] } }).toFindNothing() - }) - - it("fails to find row containing all", async () => { - await expectQuery({ - contains: { numbers: ["one", "two", "three"] }, - }).toFindNothing() - }) - - it("finds all with empty list", async () => { - await expectQuery({ contains: { numbers: [] } }).toContainExactly([ - { numbers: ["one", "two"] }, - { numbers: ["three"] }, - ]) - }) - }) - - describe("notContains", () => { - it("successfully finds a row", async () => { - await expectQuery({ - notContains: { numbers: ["one"] }, - }).toContainExactly([{ numbers: ["three"] }]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - notContains: { numbers: ["one", "two", "three"] }, - }).toContainExactly([ - { numbers: ["one", "two"] }, - { numbers: ["three"] }, - ]) - }) - - // Not sure if this is correct behaviour but changing it would be a - // breaking change. - it("finds all with empty list", async () => { - await expectQuery({ notContains: { numbers: [] } }).toContainExactly([ - { numbers: ["one", "two"] }, - { numbers: ["three"] }, - ]) - }) - }) - - describe("containsAny", () => { - it("successfully finds rows", async () => { - await expectQuery({ - containsAny: { numbers: ["one", "two", "three"] }, - }).toContainExactly([ - { numbers: ["one", "two"] }, - { numbers: ["three"] }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - containsAny: { numbers: ["none"] }, - }).toFindNothing() - }) - - it("finds all with empty list", async () => { - await expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ - { numbers: ["one", "two"] }, - { numbers: ["three"] }, - ]) - }) - }) - }) - - describe("bigints", () => { - const SMALL = "1" - const MEDIUM = "10000000" - - // Our bigints are int64s in most datasources. - let BIG = "9223372036854775807" - - beforeAll(async () => { - sourceId = await createTable({ - num: { name: "num", type: FieldType.BIGINT }, - }) - await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) - }) - - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { num: SMALL } }).toContainExactly([ - { num: SMALL }, - ]) - }) - - it("successfully finds a big value", async () => { - await expectQuery({ equal: { num: BIG } }).toContainExactly([ - { num: BIG }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { num: "2" } }).toFindNothing() - }) - }) - - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ - { num: MEDIUM }, - { num: BIG }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ - { num: SMALL }, - { num: MEDIUM }, - { num: BIG }, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ - { num: SMALL }, - ]) - }) - - it("successfully finds all rows", async () => { - await expectQuery({ - oneOf: { num: [SMALL, MEDIUM, BIG] }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { num: [2] } }).toFindNothing() - }) - }) - - // Range searches against bigints don't seem to work at all in Lucene, and I - // couldn't figure out why. Given that we're replacing Lucene with SQS, - // we've decided not to spend time on it. - !isLucene && - describe("range", () => { + describe("contains", () => { it("successfully finds a row", async () => { await expectQuery({ - range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }]) + contains: { numbers: ["one"] }, + }).toContainExactly([{ numbers: ["one", "two"] }]) }) - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { numbers: ["none"] } }).toFindNothing() }) - it("successfully finds a row with a high bound", async () => { + it("fails to find row containing all", async () => { await expectQuery({ - range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { num: { low: "5", high: "5" } }, + contains: { numbers: ["one", "two", "three"] }, }).toFindNothing() }) - it("can search using just a low value", async () => { - await expectQuery({ - range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("can search using just a high value", async () => { - await expectQuery({ - range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + it("finds all with empty list", async () => { + await expectQuery({ contains: { numbers: [] } }).toContainExactly([ + { numbers: ["one", "two"] }, + { numbers: ["three"] }, + ]) }) }) - }) - isInternal && - describe("auto", () => { - beforeAll(async () => { - sourceId = await createTable({ - auto: { - name: "auto", - type: FieldType.AUTO, - autocolumn: true, - subtype: AutoFieldSubType.AUTO_ID, - }, + describe("notContains", () => { + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { numbers: ["one"] }, + }).toContainExactly([{ numbers: ["three"] }]) }) - await createRows(new Array(10).fill({})) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + notContains: { numbers: ["one", "two", "three"] }, + }).toContainExactly([ + { numbers: ["one", "two"] }, + { numbers: ["three"] }, + ]) + }) + + // Not sure if this is correct behaviour but changing it would be a + // breaking change. + it("finds all with empty list", async () => { + await expectQuery({ notContains: { numbers: [] } }).toContainExactly([ + { numbers: ["one", "two"] }, + { numbers: ["three"] }, + ]) + }) + }) + + describe("containsAny", () => { + it("successfully finds rows", async () => { + await expectQuery({ + containsAny: { numbers: ["one", "two", "three"] }, + }).toContainExactly([ + { numbers: ["one", "two"] }, + { numbers: ["three"] }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { numbers: ["none"] }, + }).toFindNothing() + }) + + it("finds all with empty list", async () => { + await expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ + { numbers: ["one", "two"] }, + { numbers: ["three"] }, + ]) + }) + }) + }) + + describe("bigints", () => { + const SMALL = "1" + const MEDIUM = "10000000" + + // Our bigints are int64s in most datasources. + let BIG = "9223372036854775807" + + beforeAll(async () => { + tableOrViewId = await createSource({ + num: { name: "num", type: FieldType.BIGINT }, + }) + await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) }) describe("equal", () => { it("successfully finds a row", async () => { - await expectQuery({ equal: { auto: 1 } }).toContainExactly([ - { auto: 1 }, + await expectQuery({ equal: { num: SMALL } }).toContainExactly([ + { num: SMALL }, + ]) + }) + + it("successfully finds a big value", async () => { + await expectQuery({ equal: { num: BIG } }).toContainExactly([ + { num: BIG }, ]) }) it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { auto: 0 } }).toFindNothing() + await expectQuery({ equal: { num: "2" } }).toFindNothing() }) }) - describe("not equal", () => { + describe("notEqual", () => { it("successfully finds a row", async () => { - await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ - { auto: 2 }, - { auto: 3 }, - { auto: 4 }, - { auto: 5 }, - { auto: 6 }, - { auto: 7 }, - { auto: 8 }, - { auto: 9 }, - { auto: 10 }, + await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + { num: MEDIUM }, + { num: BIG }, ]) }) it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ - { auto: 1 }, - { auto: 2 }, - { auto: 3 }, - { auto: 4 }, - { auto: 5 }, - { auto: 6 }, - { auto: 7 }, - { auto: 8 }, - { auto: 9 }, - { auto: 10 }, + await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, ]) }) }) describe("oneOf", () => { it("successfully finds a row", async () => { - await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ - { auto: 1 }, + await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + { num: SMALL }, + ]) + }) + + it("successfully finds all rows", async () => { + await expectQuery({ + oneOf: { num: [SMALL, MEDIUM, BIG] }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { num: [2] } }).toFindNothing() + }) + }) + + // Range searches against bigints don't seem to work at all in Lucene, and I + // couldn't figure out why. Given that we're replacing Lucene with SQS, + // we've decided not to spend time on it. + !isLucene && + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }]) + }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing() + }) + + it("can search using just a low value", async () => { + await expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("can search using just a high value", async () => { + await expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + }) + }) + + isInternal && + describe("auto", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + auto: { + name: "auto", + type: FieldType.AUTO, + autocolumn: true, + subtype: AutoFieldSubType.AUTO_ID, + }, + }) + await createRows(new Array(10).fill({})) + }) + + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { auto: 1 } }).toContainExactly([ + { auto: 1 }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { auto: 0 } }).toFindNothing() + }) + }) + + describe("not equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ + { auto: 2 }, + { auto: 3 }, + { auto: 4 }, + { auto: 5 }, + { auto: 6 }, + { auto: 7 }, + { auto: 8 }, + { auto: 9 }, + { auto: 10 }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ + { auto: 1 }, + { auto: 2 }, + { auto: 3 }, + { auto: 4 }, + { auto: 5 }, + { auto: 6 }, + { auto: 7 }, + { auto: 8 }, + { auto: 9 }, + { auto: 10 }, + ]) + }) + }) + + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ + { auto: 1 }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { auto: [0] } }).toFindNothing() + }) + }) + + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { auto: { low: 1, high: 1 } }, + }).toContainExactly([{ auto: 1 }]) + }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { auto: { low: 1, high: 2 } }, + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { auto: { low: 2, high: 2 } }, + }).toContainExactly([{ auto: 2 }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { auto: { low: 0, high: 0 } }, + }).toFindNothing() + }) + + isSqs && + it("can search using just a low value", async () => { + await expectQuery({ + range: { auto: { low: 9 } }, + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) + + isSqs && + it("can search using just a high value", async () => { + await expectQuery({ + range: { auto: { high: 2 } }, + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) + }) + + isSqs && + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.ASCENDING, + }).toMatchExactly([ + { auto: 1 }, + { auto: 2 }, + { auto: 3 }, + { auto: 4 }, + { auto: 5 }, + { auto: 6 }, + { auto: 7 }, + { auto: 8 }, + { auto: 9 }, + { auto: 10 }, + ]) + }) + + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.DESCENDING, + }).toMatchExactly([ + { auto: 10 }, + { auto: 9 }, + { auto: 8 }, + { auto: 7 }, + { auto: 6 }, + { auto: 5 }, + { auto: 4 }, + { auto: 3 }, + { auto: 2 }, + { auto: 1 }, + ]) + }) + + // This is important for pagination. The order of results must always + // be stable or pagination will break. We don't want the user to need + // to specify an order for pagination to work. + it("is stable without a sort specified", async () => { + let { rows: fullRowList } = await config.api.row.search( + tableOrViewId, + { + tableId: tableOrViewId, + query: {}, + } + ) + + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { + const response = await config.api.row.search(tableOrViewId, { + tableId: tableOrViewId, + limit: 1, + paginate: true, + query: {}, + bookmark, + }) + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) + }) + }) + + describe("pagination", () => { + it("should paginate through all rows", async () => { + // @ts-ignore + let bookmark: string | number = undefined + let rows: Row[] = [] + + // eslint-disable-next-line no-constant-condition + while (true) { + const response = await config.api.row.search(tableOrViewId, { + tableId: tableOrViewId, + limit: 3, + query: {}, + bookmark, + paginate: true, + }) + + rows.push(...response.rows) + + if (!response.bookmark || !response.hasNextPage) { + break + } + bookmark = response.bookmark + } + + const autoValues = rows.map(row => row.auto).sort((a, b) => a - b) + expect(autoValues).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + }) + }) + }) + + describe("field name 1:name", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + "1:name": { name: "1:name", type: FieldType.STRING }, + }) + await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) + }) + + it("successfully finds a row", async () => { + await expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ + { "1:name": "bar" }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() + }) + }) + + isSql && + describe("related formulas", () => { + beforeAll(async () => { + const arrayTable = await createSource( + { + name: { name: "name", type: FieldType.STRING }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["option 1", "option 2"], + }, + }, + }, + "array" + ) + tableOrViewId = await createSource( + { + relationship: { + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_ONE, + name: "relationship", + fieldName: "relate", + tableId: arrayTable, + constraints: { + type: "array", + }, + }, + formula: { + type: FieldType.FORMULA, + name: "formula", + formula: encodeJSBinding( + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` + ), + }, + }, + "main" + ) + const arrayRows = await Promise.all([ + config.api.row.save(arrayTable, { + name: "foo", + array: ["option 1"], + }), + config.api.row.save(arrayTable, { + name: "bar", + array: ["option 2"], + }), + ]) + await Promise.all([ + config.api.row.save(tableOrViewId, { + relationship: [arrayRows[0]._id, arrayRows[1]._id], + }), + ]) + }) + + it("formula is correct with relationship arrays", async () => { + await expectQuery({}).toContain([{ formula: "option 1,option 2" }]) + }) + }) + + describe("user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + tableOrViewId = await createSource({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + }) + + await createRows([{ user: user1 }, { user: user2 }, { user: null }]) + }) + + describe("equal", () => { + it("successfully finds a row", async () => { + await expectQuery({ equal: { user: user1._id } }).toContainExactly([ + { user: { _id: user1._id } }, ]) }) it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { auto: [0] } }).toFindNothing() + await expectQuery({ equal: { user: "us_none" } }).toFindNothing() }) }) - describe("range", () => { + describe("notEqual", () => { it("successfully finds a row", async () => { - await expectQuery({ - range: { auto: { low: 1, high: 1 } }, - }).toContainExactly([{ auto: 1 }]) + await expectQuery({ notEqual: { user: user1._id } }).toContainExactly( + [{ user: { _id: user2._id } }, {}] + ) }) - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { auto: { low: 1, high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }]) - }) - - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { auto: { low: 2, high: 2 } }, - }).toContainExactly([{ auto: 2 }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { auto: { low: 0, high: 0 } }, - }).toFindNothing() - }) - - isSqs && - it("can search using just a low value", async () => { - await expectQuery({ - range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }]) - }) - - isSqs && - it("can search using just a high value", async () => { - await expectQuery({ - range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }]) - }) - }) - - isSqs && - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([ - { auto: 1 }, - { auto: 2 }, - { auto: 3 }, - { auto: 4 }, - { auto: 5 }, - { auto: 6 }, - { auto: 7 }, - { auto: 8 }, - { auto: 9 }, - { auto: 10 }, - ]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([ - { auto: 10 }, - { auto: 9 }, - { auto: 8 }, - { auto: 7 }, - { auto: 6 }, - { auto: 5 }, - { auto: 4 }, - { auto: 3 }, - { auto: 2 }, - { auto: 1 }, - ]) - }) - - // This is important for pagination. The order of results must always - // be stable or pagination will break. We don't want the user to need - // to specify an order for pagination to work. - it("is stable without a sort specified", async () => { - let { rows: fullRowList } = await config.api.row.search(sourceId, { - tableId: sourceId, - query: {}, - }) - - // repeat the search many times to check the first row is always the same - let bookmark: string | number | undefined, - hasNextPage: boolean | undefined = true, - rowCount: number = 0 - do { - const response = await config.api.row.search(sourceId, { - tableId: sourceId, - limit: 1, - paginate: true, - query: {}, - bookmark, - }) - bookmark = response.bookmark - hasNextPage = response.hasNextPage - expect(response.rows.length).toEqual(1) - const foundRow = response.rows[0] - expect(foundRow).toEqual(fullRowList[rowCount++]) - } while (hasNextPage) - }) - }) - - describe("pagination", () => { - it("should paginate through all rows", async () => { - // @ts-ignore - let bookmark: string | number = undefined - let rows: Row[] = [] - - // eslint-disable-next-line no-constant-condition - while (true) { - const response = await config.api.row.search(sourceId, { - tableId: sourceId, - limit: 3, - query: {}, - bookmark, - paginate: true, - }) - - rows.push(...response.rows) - - if (!response.bookmark || !response.hasNextPage) { - break - } - bookmark = response.bookmark - } - - const autoValues = rows.map(row => row.auto).sort((a, b) => a - b) - expect(autoValues).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - }) - }) - }) - - describe("field name 1:name", () => { - beforeAll(async () => { - sourceId = await createTable({ - "1:name": { name: "1:name", type: FieldType.STRING }, - }) - await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) - }) - - it("successfully finds a row", async () => { - await expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ - { "1:name": "bar" }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() - }) - }) - - isSql && - describe("related formulas", () => { - beforeAll(async () => { - const arrayTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - array: { - name: "array", - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - inclusion: ["option 1", "option 2"], - }, - }, - }, - "array" - ) - sourceId = await createTable( - { - relationship: { - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - name: "relationship", - fieldName: "relate", - tableId: arrayTable, - constraints: { - type: "array", - }, - }, - formula: { - type: FieldType.FORMULA, - name: "formula", - formula: encodeJSBinding( - `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` - ), - }, - }, - "main" - ) - const arrayRows = await Promise.all([ - config.api.row.save(arrayTable, { - name: "foo", - array: ["option 1"], - }), - config.api.row.save(arrayTable, { - name: "bar", - array: ["option 2"], - }), - ]) - await Promise.all([ - config.api.row.save(sourceId, { - relationship: [arrayRows[0]._id, arrayRows[1]._id], - }), - ]) - }) - - it("formula is correct with relationship arrays", async () => { - await expectQuery({}).toContain([{ formula: "option 1,option 2" }]) - }) - }) - - describe("user", () => { - let user1: User - let user2: User - - beforeAll(async () => { - user1 = await config.createUser({ _id: `us_${utils.newid()}` }) - user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - - sourceId = await createTable({ - user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - }) - - await createRows([ - { user: JSON.stringify(user1) }, - { user: JSON.stringify(user2) }, - { user: null }, - ]) - }) - - describe("equal", () => { - it("successfully finds a row", async () => { - await expectQuery({ equal: { user: user1._id } }).toContainExactly([ - { user: { _id: user1._id } }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ equal: { user: "us_none" } }).toFindNothing() - }) - }) - - describe("notEqual", () => { - it("successfully finds a row", async () => { - await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ - { user: { _id: user2._id } }, - {}, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ - { user: { _id: user1._id } }, - { user: { _id: user2._id } }, - {}, - ]) - }) - }) - - describe("oneOf", () => { - it("successfully finds a row", async () => { - await expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ - { user: { _id: user1._id } }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing() - }) - }) - - describe("empty", () => { - it("finds empty rows", async () => { - await expectQuery({ empty: { user: null } }).toContainExactly([{}]) - }) - }) - - describe("notEmpty", () => { - it("finds non-empty rows", async () => { - await expectQuery({ notEmpty: { user: null } }).toContainExactly([ - { user: { _id: user1._id } }, - { user: { _id: user2._id } }, - ]) - }) - }) - }) - - describe("multi user", () => { - let user1: User - let user2: User - - beforeAll(async () => { - user1 = await config.createUser({ _id: `us_${utils.newid()}` }) - user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - - sourceId = await createTable({ - users: { - name: "users", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - constraints: { type: "array" }, - }, - number: { - name: "number", - type: FieldType.NUMBER, - }, - }) - - await createRows([ - { number: 1, users: JSON.stringify([user1]) }, - { number: 2, users: JSON.stringify([user2]) }, - { number: 3, users: JSON.stringify([user1, user2]) }, - { number: 4, users: JSON.stringify([]) }, - ]) - }) - - describe("contains", () => { - it("successfully finds a row", async () => { - await expectQuery({ - contains: { users: [user1._id] }, - }).toContainExactly([ - { users: [{ _id: user1._id }] }, - { users: [{ _id: user1._id }, { _id: user2._id }] }, - ]) - }) - - it("successfully finds a row searching with a string", async () => { - await expectQuery({ - // @ts-expect-error this test specifically goes against the type to - // test that we coerce the string to an array. - contains: { "1:users": user1._id }, - }).toContainExactly([ - { users: [{ _id: user1._id }] }, - { users: [{ _id: user1._id }, { _id: user2._id }] }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() - }) - }) - - describe("notContains", () => { - it("successfully finds a row", async () => { - await expectQuery({ - notContains: { users: [user1._id] }, - }).toContainExactly([{ users: [{ _id: user2._id }] }, {}]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - notContains: { users: ["us_none"] }, - }).toContainExactly([ - { users: [{ _id: user1._id }] }, - { users: [{ _id: user2._id }] }, - { users: [{ _id: user1._id }, { _id: user2._id }] }, - {}, - ]) - }) - }) - - describe("containsAny", () => { - it("successfully finds rows", async () => { - await expectQuery({ - containsAny: { users: [user1._id, user2._id] }, - }).toContainExactly([ - { users: [{ _id: user1._id }] }, - { users: [{ _id: user2._id }] }, - { users: [{ _id: user1._id }, { _id: user2._id }] }, - ]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - containsAny: { users: ["us_none"] }, - }).toFindNothing() - }) - }) - - describe("multi-column equals", () => { - it("successfully finds a row", async () => { - await expectQuery({ - equal: { number: 1 }, - contains: { users: [user1._id] }, - }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) - }) - - it("fails to find nonexistent row", async () => { - await expectQuery({ - equal: { number: 2 }, - contains: { users: [user1._id] }, - }).toFindNothing() - }) - }) - }) - - // This will never work for Lucene. - !isLucene && - // It also can't work for in-memory searching because the related table name - // isn't available. - !isInMemory && - describe("relations", () => { - let productCategoryTable: Table, productCatRows: Row[] - - beforeAll(async () => { - const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.ONE_TO_MANY - ) - sourceId = tableId - productCategoryTable = relatedTable - - productCatRows = await Promise.all([ - config.api.row.save(productCategoryTable._id!, { name: "foo" }), - config.api.row.save(productCategoryTable._id!, { name: "bar" }), - ]) - - await Promise.all([ - config.api.row.save(sourceId, { - name: "foo", - productCat: [productCatRows[0]._id], - }), - config.api.row.save(sourceId, { - name: "bar", - productCat: [productCatRows[1]._id], - }), - config.api.row.save(sourceId, { - name: "baz", - productCat: [], - }), - ]) - }) - - it("should be able to filter by relationship using column name", async () => { - await expectQuery({ - equal: { ["productCat.name"]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) - - it("should be able to filter by relationship using table name", async () => { - await expectQuery({ - equal: { [`${productCategoryTable.name}.name`]: "foo" }, - }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - ]) - }) - - it("shouldn't return any relationship for last row", async () => { - await expectQuery({ - equal: { ["name"]: "baz" }, - }).toContainExactly([{ name: "baz", productCat: undefined }]) - }) - }) - - isSql && - describe("big relations", () => { - beforeAll(async () => { - const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.MANY_TO_ONE - ) - sourceId = tableId - const mainRow = await config.api.row.save(sourceId, { - name: "foo", - }) - for (let i = 0; i < 11; i++) { - await config.api.row.save(relatedTable._id!, { - name: i, - product: [mainRow._id!], - }) - } - }) - - it("can only pull 10 related rows", async () => { - await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { - const response = await expectQuery({}).toContain([{ name: "foo" }]) - expect(response.rows[0].productCat).toBeArrayOfSize(10) + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly( + [{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}] + ) }) }) - it("can pull max rows when env not set (defaults to 500)", async () => { - const response = await expectQuery({}).toContain([{ name: "foo" }]) - expect(response.rows[0].productCat).toBeArrayOfSize(11) - }) - }) - ;(isSqs || isLucene) && - describe("relations to same table", () => { - let relatedTable: string, relatedRows: Row[] - - beforeAll(async () => { - relatedTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) - sourceId = await createTable({ - name: { name: "name", type: FieldType.STRING }, - related1: { - type: FieldType.LINK, - name: "related1", - fieldName: "main1", - tableId: relatedTable, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - related2: { - type: FieldType.LINK, - name: "related2", - fieldName: "main2", - tableId: relatedTable, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - }) - relatedRows = await Promise.all([ - config.api.row.save(relatedTable, { name: "foo" }), - config.api.row.save(relatedTable, { name: "bar" }), - config.api.row.save(relatedTable, { name: "baz" }), - config.api.row.save(relatedTable, { name: "boo" }), - ]) - await Promise.all([ - config.api.row.save(sourceId, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], - }), - config.api.row.save(sourceId, { - name: "test2", - related1: [relatedRows[2]._id!], - related2: [relatedRows[3]._id!], - }), - ]) - }) - - it("should be able to relate to same table", async () => { - await expectSearch({ - query: {}, - }).toContainExactly([ - { - name: "test", - related1: [{ _id: relatedRows[0]._id }], - related2: [{ _id: relatedRows[1]._id }], - }, - { - name: "test2", - related1: [{ _id: relatedRows[2]._id }], - related2: [{ _id: relatedRows[3]._id }], - }, - ]) - }) - - isSqs && - it("should be able to filter down to second row with equal", async () => { - await expectSearch({ - query: { - equal: { - ["related1.name"]: "baz", - }, - }, - }).toContainExactly([ - { - name: "test2", - related1: [{ _id: relatedRows[2]._id }], - }, + describe("oneOf", () => { + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + { user: { _id: user1._id } }, ]) }) - isSqs && - it("should be able to filter down to first row with not equal", async () => { - await expectSearch({ - query: { - notEqual: { - ["1:related2.name"]: "bar", - ["2:related2.name"]: "baz", - ["3:related2.name"]: "boo", - }, + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing() + }) + }) + + describe("empty", () => { + it("finds empty rows", async () => { + await expectQuery({ empty: { user: null } }).toContainExactly([{}]) + }) + }) + + describe("notEmpty", () => { + it("finds non-empty rows", async () => { + await expectQuery({ notEmpty: { user: null } }).toContainExactly([ + { user: { _id: user1._id } }, + { user: { _id: user2._id } }, + ]) + }) + }) + }) + + describe("multi user", () => { + let user1: User + let user2: User + + beforeAll(async () => { + user1 = await config.createUser({ _id: `us_${utils.newid()}` }) + user2 = await config.createUser({ _id: `us_${utils.newid()}` }) + + tableOrViewId = await createSource({ + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { type: "array" }, + }, + number: { + name: "number", + type: FieldType.NUMBER, + }, + }) + + await createRows([ + { number: 1, users: [user1] }, + { number: 2, users: [user2] }, + { number: 3, users: [user1, user2] }, + { number: 4, users: [] }, + ]) + }) + + describe("contains", () => { + it("successfully finds a row", async () => { + await expectQuery({ + contains: { users: [user1._id] }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ]) + }) + + it("successfully finds a row searching with a string", async () => { + await expectQuery({ + // @ts-expect-error this test specifically goes against the type to + // test that we coerce the string to an array. + contains: { "1:users": user1._id }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + contains: { users: ["us_none"] }, + }).toFindNothing() + }) + }) + + describe("notContains", () => { + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user2._id }] }, {}]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + notContains: { users: ["us_none"] }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + {}, + ]) + }) + }) + + describe("containsAny", () => { + it("successfully finds rows", async () => { + await expectQuery({ + containsAny: { users: [user1._id, user2._id] }, + }).toContainExactly([ + { users: [{ _id: user1._id }] }, + { users: [{ _id: user2._id }] }, + { users: [{ _id: user1._id }, { _id: user2._id }] }, + ]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { users: ["us_none"] }, + }).toFindNothing() + }) + }) + + describe("multi-column equals", () => { + it("successfully finds a row", async () => { + await expectQuery({ + equal: { number: 1 }, + contains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) + }) + + it("fails to find nonexistent row", async () => { + await expectQuery({ + equal: { number: 2 }, + contains: { users: [user1._id] }, + }).toFindNothing() + }) + }) + }) + + // This will never work for Lucene. + !isLucene && + // It also can't work for in-memory searching because the related table name + // isn't available. + !isInMemory && + describe("relations", () => { + let productCategoryTable: Table, productCatRows: Row[] + + beforeAll(async () => { + const { relatedTable, tableId } = await basicRelationshipTables( + RelationshipType.ONE_TO_MANY + ) + tableOrViewId = tableId + productCategoryTable = relatedTable + + productCatRows = await Promise.all([ + config.api.row.save(productCategoryTable._id!, { name: "foo" }), + config.api.row.save(productCategoryTable._id!, { name: "bar" }), + ]) + + await Promise.all([ + config.api.row.save(tableOrViewId, { + name: "foo", + productCat: [productCatRows[0]._id], + }), + config.api.row.save(tableOrViewId, { + name: "bar", + productCat: [productCatRows[1]._id], + }), + config.api.row.save(tableOrViewId, { + name: "baz", + productCat: [], + }), + ]) + }) + + it("should be able to filter by relationship using column name", async () => { + await expectQuery({ + equal: { ["productCat.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should be able to filter by relationship using table name", async () => { + await expectQuery({ + equal: { [`${productCategoryTable.name}.name`]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("shouldn't return any relationship for last row", async () => { + await expectQuery({ + equal: { ["name"]: "baz" }, + }).toContainExactly([{ name: "baz", productCat: undefined }]) + }) + }) + + isSql && + describe("big relations", () => { + beforeAll(async () => { + const { relatedTable, tableId } = await basicRelationshipTables( + RelationshipType.MANY_TO_ONE + ) + tableOrViewId = tableId + const mainRow = await config.api.row.save(tableOrViewId, { + name: "foo", + }) + for (let i = 0; i < 11; i++) { + await config.api.row.save(relatedTable._id!, { + name: i, + product: [mainRow._id!], + }) + } + }) + + it("can only pull 10 related rows", async () => { + await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(10) + }) + }) + + it("can pull max rows when env not set (defaults to 500)", async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(11) + }) + }) + ;(isSqs || isLucene) && + describe("relations to same table", () => { + let relatedTable: string, relatedRows: Row[] + + beforeAll(async () => { + relatedTable = await createSource( + { + name: { name: "name", type: FieldType.STRING }, }, + "productCategory" + ) + tableOrViewId = await createSource({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + related2: { + type: FieldType.LINK, + name: "related2", + fieldName: "main2", + tableId: relatedTable, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable, { name: "foo" }), + config.api.row.save(relatedTable, { name: "bar" }), + config.api.row.save(relatedTable, { name: "baz" }), + config.api.row.save(relatedTable, { name: "boo" }), + ]) + await Promise.all([ + config.api.row.save(tableOrViewId, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }), + config.api.row.save(tableOrViewId, { + name: "test2", + related1: [relatedRows[2]._id!], + related2: [relatedRows[3]._id!], + }), + ]) + }) + + it("should be able to relate to same table", async () => { + await expectSearch({ + query: {}, }).toContainExactly([ { name: "test", related1: [{ _id: relatedRows[0]._id }], + related2: [{ _id: relatedRows[1]._id }], + }, + { + name: "test2", + related1: [{ _id: relatedRows[2]._id }], + related2: [{ _id: relatedRows[3]._id }], }, ]) }) - }) - isInternal && - describe("no column error backwards compat", () => { + isSqs && + it("should be able to filter down to second row with equal", async () => { + await expectSearch({ + query: { + equal: { + ["related1.name"]: "baz", + }, + }, + }).toContainExactly([ + { + name: "test2", + related1: [{ _id: relatedRows[2]._id }], + }, + ]) + }) + + isSqs && + it("should be able to filter down to first row with not equal", async () => { + await expectSearch({ + query: { + notEqual: { + ["1:related2.name"]: "bar", + ["2:related2.name"]: "baz", + ["3:related2.name"]: "boo", + }, + }, + }).toContainExactly([ + { + name: "test", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) + }) + + isInternal && + describe("no column error backwards compat", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + }) + + it("shouldn't error when column doesn't exist", async () => { + await expectSearch({ + query: { + string: { + "1:something": "a", + }, + }, + }).toMatch({ rows: [] }) + }) + }) + + // lucene can't count the total rows + !isLucene && + describe("row counting", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + await createRows([{ name: "a" }, { name: "b" }]) + }) + + it("should be able to count rows when option set", async () => { + await expectSearch({ + countRows: true, + query: { + notEmpty: { + name: true, + }, + }, + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) + + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ + countRows: false, + query: { + notEmpty: { + name: true, + }, + }, + }).toNotHaveProperty(["totalRows"]) + }) + }) + + describe("Invalid column definitions", () => { beforeAll(async () => { - sourceId = await createTable({ + // need to create an invalid table - means ignoring typescript + tableOrViewId = await createSource({ + // @ts-ignore + invalid: { + type: FieldType.STRING, + }, name: { name: "name", type: FieldType.STRING, }, }) + await createRows([ + { name: "foo", invalid: "id1" }, + { name: "bar", invalid: "id2" }, + ]) }) - it("shouldn't error when column doesn't exist", async () => { + it("can get rows with all table data", async () => { await expectSearch({ - query: { - string: { - "1:something": "a", - }, - }, - }).toMatch({ rows: [] }) + query: {}, + }).toContain([ + { name: "foo", invalid: "id1" }, + { name: "bar", invalid: "id2" }, + ]) }) }) - // lucene can't count the total rows - !isLucene && - describe("row counting", () => { + describe.each(["data_name_test", "name_data_test", "name_test_data_"])( + "special (%s) case", + column => { + beforeAll(async () => { + tableOrViewId = await createSource({ + [column]: { + name: column, + type: FieldType.STRING, + }, + }) + await createRows([{ [column]: "a" }, { [column]: "b" }]) + }) + + it("should be able to query a column with data_ in it", async () => { + await expectSearch({ + query: { + equal: { + [`1:${column}`]: "a", + }, + }, + }).toContainExactly([{ [column]: "a" }]) + }) + } + ) + + isInternal && + describe("sample data", () => { + beforeAll(async () => { + await config.api.application.addSampleData(config.appId!) + tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id! + rows = await config.api.row.fetch(tableOrViewId) + }) + + it("should be able to search sample data", async () => { + await expectSearch({ + query: {}, + }).toContain([ + { + "First Name": "Mandy", + }, + ]) + }) + }) + + describe.each([ + { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, + { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, + { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + ])("date special cases", ({ low, high }) => { + const earlyDate = "2024-07-03T10:00:00.000Z", + laterDate = "2024-07-03T11:00:00.000Z" beforeAll(async () => { - sourceId = await createTable({ - name: { - name: "name", + tableOrViewId = await createSource({ + date: { + name: "date", + type: FieldType.DATETIME, + }, + }) + await createRows([{ date: earlyDate }, { date: laterDate }]) + }) + + it("should be able to handle a date search", async () => { + await expectSearch({ + query: { + range: { + "1:date": { low, high }, + }, + }, + }).toContainExactly([{ date: earlyDate }, { date: laterDate }]) + }) + }) + + describe.each([ + "名前", // Japanese for "name" + "Benutzer-ID", // German for "user ID", includes a hyphen + "numéro", // French for "number", includes an accent + "år", // Swedish for "year", includes a ring above + "naïve", // English word borrowed from French, includes an umlaut + "الاسم", // Arabic for "name" + "оплата", // Russian for "payment" + "पता", // Hindi for "address" + "用戶名", // Chinese for "username" + "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + "preço", // Portuguese for "price", includes a cedilla + "사용자명", // Korean for "username" + "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + "файл", // Bulgarian for "file" + "δεδομένα", // Greek for "data" + "geändert_am", // German for "modified on", includes an umlaut + "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + "São_Paulo", // Portuguese, includes an underscore and a tilde + "età", // Italian for "age", includes an accent + "ชื่อผู้ใช้", // Thai for "username" + ])("non-ascii column name: %s", name => { + beforeAll(async () => { + tableOrViewId = await createSource({ + [name]: { + name, type: FieldType.STRING, }, }) - await createRows([{ name: "a" }, { name: "b" }]) + await createRows([{ [name]: "a" }, { [name]: "b" }]) }) - it("should be able to count rows when option set", async () => { - await expectSearch({ - countRows: true, - query: { - notEmpty: { - name: true, - }, - }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) }) - }) - - it("shouldn't count rows when option is not set", async () => { - await expectSearch({ - countRows: false, - query: { - notEmpty: { - name: true, - }, - }, - }).toNotHaveProperty(["totalRows"]) - }) - }) - - describe("Invalid column definitions", () => { - beforeAll(async () => { - // need to create an invalid table - means ignoring typescript - sourceId = await createTable({ - // @ts-ignore - invalid: { - type: FieldType.STRING, - }, - name: { - name: "name", - type: FieldType.STRING, - }, - }) - await createRows([ - { name: "foo", invalid: "id1" }, - { name: "bar", invalid: "id2" }, - ]) - }) - - it("can get rows with all table data", async () => { - await expectSearch({ - query: {}, - }).toContain([ - { name: "foo", invalid: "id1" }, - { name: "bar", invalid: "id2" }, - ]) - }) - }) - - describe.each(["data_name_test", "name_data_test", "name_test_data_"])( - "special (%s) case", - column => { - beforeAll(async () => { - sourceId = await createTable({ - [column]: { - name: column, - type: FieldType.STRING, - }, - }) - await createRows([{ [column]: "a" }, { [column]: "b" }]) - }) - - it("should be able to query a column with data_ in it", async () => { + it("should be able to query a column with non-ascii characters", async () => { await expectSearch({ query: { equal: { - [`1:${column}`]: "a", + [`1:${name}`]: "a", }, }, - }).toContainExactly([{ [column]: "a" }]) - }) - } - ) - - isInternal && - describe("sample data", () => { - beforeAll(async () => { - await config.api.application.addSampleData(config.appId!) - sourceId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id! - rows = await config.api.row.fetch(sourceId) - }) - - it("should be able to search sample data", async () => { - await expectSearch({ - query: {}, - }).toContain([ - { - "First Name": "Mandy", - }, - ]) + }).toContainExactly([{ [name]: "a" }]) }) }) - describe.each([ - { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, - { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, - { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, - { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, - ])("date special cases", ({ low, high }) => { - const earlyDate = "2024-07-03T10:00:00.000Z", - laterDate = "2024-07-03T11:00:00.000Z" - beforeAll(async () => { - sourceId = await createTable({ - date: { - name: "date", - type: FieldType.DATETIME, - }, - }) - await createRows([{ date: earlyDate }, { date: laterDate }]) - }) - - it("should be able to handle a date search", async () => { - await expectSearch({ - query: { - range: { - "1:date": { low, high }, - }, - }, - }).toContainExactly([{ date: earlyDate }, { date: laterDate }]) - }) - }) - - describe.each([ - "名前", // Japanese for "name" - "Benutzer-ID", // German for "user ID", includes a hyphen - "numéro", // French for "number", includes an accent - "år", // Swedish for "year", includes a ring above - "naïve", // English word borrowed from French, includes an umlaut - "الاسم", // Arabic for "name" - "оплата", // Russian for "payment" - "पता", // Hindi for "address" - "用戶名", // Chinese for "username" - "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla - "preço", // Portuguese for "price", includes a cedilla - "사용자명", // Korean for "username" - "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" - "файл", // Bulgarian for "file" - "δεδομένα", // Greek for "data" - "geändert_am", // German for "modified on", includes an umlaut - "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore - "São_Paulo", // Portuguese, includes an underscore and a tilde - "età", // Italian for "age", includes an accent - "ชื่อผู้ใช้", // Thai for "username" - ])("non-ascii column name: %s", name => { - beforeAll(async () => { - sourceId = await createTable({ - [name]: { - name, - type: FieldType.STRING, - }, - }) - await createRows([{ [name]: "a" }, { [name]: "b" }]) - }) - - it("should be able to query a column with non-ascii characters", async () => { - await expectSearch({ - query: { - equal: { - [`1:${name}`]: "a", - }, - }, - }).toContainExactly([{ [name]: "a" }]) - }) - }) - - // This is currently not supported in external datasources, it produces SQL - // errors at time of writing. We supported it (potentially by accident) in - // Lucene, though, so we need to make sure it's supported in SQS as well. We - // found real cases in production of column names ending in a space. - isInternal && - describe("space at end of column name", () => { - beforeAll(async () => { - sourceId = await createTable({ - "name ": { - name: "name ", - type: FieldType.STRING, - }, - }) - await createRows([{ ["name "]: "foo" }, { ["name "]: "bar" }]) - }) - - it("should be able to query a column that ends with a space", async () => { - await expectSearch({ - query: { - string: { - "name ": "foo", - }, - }, - }).toContainExactly([{ ["name "]: "foo" }]) - }) - - it("should be able to query a column that ends with a space using numeric notation", async () => { - await expectSearch({ - query: { - string: { - "1:name ": "foo", - }, - }, - }).toContainExactly([{ ["name "]: "foo" }]) - }) - }) - - // This was never actually supported in Lucene but SQS does support it, so may - // as well have a test for it. - ;(isSqs || isInMemory) && - describe("space at start of column name", () => { - beforeAll(async () => { - sourceId = await createTable({ - " name": { - name: " name", - type: FieldType.STRING, - }, - }) - await createRows([{ [" name"]: "foo" }, { [" name"]: "bar" }]) - }) - - it("should be able to query a column that starts with a space", async () => { - await expectSearch({ - query: { - string: { - " name": "foo", - }, - }, - }).toContainExactly([{ [" name"]: "foo" }]) - }) - - it("should be able to query a column that starts with a space using numeric notation", async () => { - await expectSearch({ - query: { - string: { - "1: name": "foo", - }, - }, - }).toContainExactly([{ [" name"]: "foo" }]) - }) - }) - - isSqs && - describe("duplicate columns", () => { - beforeAll(async () => { - sourceId = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - await context.doInAppContext(config.getAppId(), async () => { - const db = context.getAppDB() - const tableDoc = await db.get
(sourceId) - tableDoc.schema.Name = { - name: "Name", - type: FieldType.STRING, - } - try { - // remove the SQLite definitions so that they can be rebuilt as part of the search - const sqliteDoc = await db.get(SQLITE_DESIGN_DOC_ID) - await db.remove(sqliteDoc) - } catch (err) { - // no-op - } - }) - await createRows([{ name: "foo", Name: "bar" }]) - }) - - it("should handle invalid duplicate column names", async () => { - await expectSearch({ - query: {}, - }).toContainExactly([{ name: "foo" }]) - }) - }) - - !isInMemory && - describe("search by _id", () => { - let row: Row - - beforeAll(async () => { - const toRelateTable = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - sourceId = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - rel: { - name: "rel", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_MANY, - tableId: toRelateTable, - fieldName: "rel", - }, - }) - const [row1, row2] = await Promise.all([ - config.api.row.save(toRelateTable, { name: "tag 1" }), - config.api.row.save(toRelateTable, { name: "tag 2" }), - ]) - row = await config.api.row.save(sourceId, { - name: "product 1", - rel: [row1._id, row2._id], - }) - }) - - it("can filter by the row ID with limit 1", async () => { - await expectSearch({ - query: { - equal: { _id: row._id }, - }, - limit: 1, - }).toContainExactly([row]) - }) - }) - - !isInternal && - describe("search by composite key", () => { - beforeAll(async () => { - const table = await config.api.table.save( - tableForDatasource(datasource, { - schema: { - idColumn1: { - name: "idColumn1", - type: FieldType.NUMBER, - }, - idColumn2: { - name: "idColumn2", - type: FieldType.NUMBER, - }, - }, - primary: ["idColumn1", "idColumn2"], - }) - ) - sourceId = table._id! - await createRows([{ idColumn1: 1, idColumn2: 2 }]) - }) - - it("can filter by the row ID with limit 1", async () => { - await expectSearch({ - query: { - equal: { _id: generateRowIdField([1, 2]) }, - }, - limit: 1, - }).toContain([ - { - idColumn1: 1, - idColumn2: 2, - }, - ]) - }) - }) - - isSql && - describe("primaryDisplay", () => { - beforeAll(async () => { - let toRelateTableId = await createTable({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - const table = await config.api.table.save( - tableForDatasource(datasource, { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - link: { - name: "link", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTableId, - fieldName: "link", - }, + // This is currently not supported in external datasources, it produces SQL + // errors at time of writing. We supported it (potentially by accident) in + // Lucene, though, so we need to make sure it's supported in SQS as well. We + // found real cases in production of column names ending in a space. + isInternal && + describe("space at end of column name", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + "name ": { + name: "name ", + type: FieldType.STRING, }, }) - ) - sourceId = table._id! - const toRelateTable = await config.api.table.get(toRelateTableId) - await config.api.table.save({ - ...toRelateTable, - primaryDisplay: "link", + await createRows([{ ["name "]: "foo" }, { ["name "]: "bar" }]) }) - const relatedRows = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "test" }), - ]) - await Promise.all([ - config.api.row.save(sourceId, { - name: "test", - link: relatedRows.map(row => row._id), - }), - ]) - }) - it("should be able to query, primary display on related table shouldn't be used", async () => { - // this test makes sure that if a relationship has been specified as the primary display on a table - // it is ignored and another column is used instead - await expectQuery({}).toContain([ - { name: "test", link: [{ primaryDisplay: "test" }] }, - ]) - }) - }) - - !isLucene && - describe("$and", () => { - beforeAll(async () => { - sourceId = await createTable({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, + it("should be able to query a column that ends with a space", async () => { + await expectSearch({ + query: { + string: { + "name ": "foo", + }, + }, + }).toContainExactly([{ ["name "]: "foo" }]) + }) + + it("should be able to query a column that ends with a space using numeric notation", async () => { + await expectSearch({ + query: { + string: { + "1:name ": "foo", + }, + }, + }).toContainExactly([{ ["name "]: "foo" }]) }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) }) - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) + // This was never actually supported in Lucene but SQS does support it, so may + // as well have a test for it. + ;(isSqs || isInMemory) && + describe("space at start of column name", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + " name": { + name: " name", + type: FieldType.STRING, + }, + }) + await createRows([{ [" name"]: "foo" }, { [" name"]: "bar" }]) + }) + + it("should be able to query a column that starts with a space", async () => { + await expectSearch({ + query: { + string: { + " name": "foo", + }, + }, + }).toContainExactly([{ [" name"]: "foo" }]) + }) + + it("should be able to query a column that starts with a space using numeric notation", async () => { + await expectSearch({ + query: { + string: { + "1: name": "foo", + }, + }, + }).toContainExactly([{ [" name"]: "foo" }]) + }) }) - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) + isSqs && + describe("duplicate columns", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + await context.doInAppContext(config.getAppId(), async () => { + const db = context.getAppDB() + const tableDoc = await db.get
(tableOrViewId) + tableDoc.schema.Name = { + name: "Name", + type: FieldType.STRING, + } + try { + // remove the SQLite definitions so that they can be rebuilt as part of the search + const sqliteDoc = await db.get(SQLITE_DESIGN_DOC_ID) + await db.remove(sqliteDoc) + } catch (err) { + // no-op + } + }) + await createRows([{ name: "foo", Name: "bar" }]) + }) + + it("should handle invalid duplicate column names", async () => { + await expectSearch({ + query: {}, + }).toContainExactly([{ name: "foo" }]) + }) }) - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Ja" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) + !isInMemory && + describe("search by _id", () => { + let row: Row + + beforeAll(async () => { + const toRelateTable = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + tableOrViewId = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + rel: { + name: "rel", + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_MANY, + tableId: toRelateTable, + fieldName: "rel", + }, + }) + const [row1, row2] = await Promise.all([ + config.api.row.save(toRelateTable, { name: "tag 1" }), + config.api.row.save(toRelateTable, { name: "tag 2" }), + ]) + row = await config.api.row.save(tableOrViewId, { + name: "product 1", + rel: [row1._id, row2._id], + }) + }) + + it("can filter by the row ID with limit 1", async () => { + await expectSearch({ + query: { + equal: { _id: row._id }, + }, + limit: 1, + }).toContainExactly([row]) + }) }) - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - range: { age: { low: 1, high: 10 } }, - }, - { string: { name: "Ja" } }, - ], + !isInternal && + describe("search by composite key", () => { + beforeAll(async () => { + const table = await config.api.table.save( + tableForDatasource(datasource, { + schema: { + idColumn1: { + name: "idColumn1", + type: FieldType.NUMBER, + }, + idColumn2: { + name: "idColumn2", + type: FieldType.NUMBER, }, - equal: { name: "Jane" }, }, - ], - }, - }).toContainExactly([{ age: 1, name: "Jane" }]) - }) - - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toFindNothing() - }) - - !isInMemory && - it("validates conditions that are not objects", async () => { - await expect( - expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], - }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1]" must be of type object' + primary: ["idColumn1", "idColumn2"], + }) ) + tableOrViewId = table._id! + await createRows([{ idColumn1: 1, idColumn2: 2 }]) }) - !isInMemory && - it("validates $and without conditions", async () => { - await expect( - expectQuery({ - $and: { - conditions: [ - { equal: { age: 10 } }, - { - $and: { - conditions: undefined as any, - }, + it("can filter by the row ID with limit 1", async () => { + await expectSearch({ + query: { + equal: { _id: generateRowIdField([1, 2]) }, + }, + limit: 1, + }).toContain([ + { + idColumn1: 1, + idColumn2: 2, + }, + ]) + }) + }) + + isSql && + describe("primaryDisplay", () => { + beforeAll(async () => { + let toRelateTableId = await createSource({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + const table = await config.api.table.save( + tableForDatasource(datasource, { + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + link: { + name: "link", + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: toRelateTableId, + fieldName: "link", + }, + }, + }) + ) + tableOrViewId = table._id! + const toRelateTable = await config.api.table.get(toRelateTableId) + await config.api.table.save({ + ...toRelateTable, + primaryDisplay: "link", + }) + const relatedRows = await Promise.all([ + config.api.row.save(toRelateTable._id!, { name: "test" }), + ]) + await Promise.all([ + config.api.row.save(tableOrViewId, { + name: "test", + link: relatedRows.map(row => row._id), + }), + ]) + }) + + it("should be able to query, primary display on related table shouldn't be used", async () => { + // this test makes sure that if a relationship has been specified as the primary display on a table + // it is ignored and another column is used instead + await expectQuery({}).toContain([ + { name: "test", link: [{ primaryDisplay: "test" }] }, + ]) + }) + }) + + !isLucene && + describe("$and", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Ja" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 10 } }, + }, + { string: { name: "Ja" } }, + ], }, - ], - }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' - ) - }) - - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $and: { - conditions: [{ equal: { name: "" } }], - }, - }, - }).toFindNothing() - }) - - it("returns all rows when onEmptyFilter set to all", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - $and: { - conditions: [{ equal: { name: "" } }], - }, - }, - }).toHaveLength(4) - }) - }) - - !isLucene && - describe("$or", () => { - beforeAll(async () => { - sourceId = await createTable({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Jan" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - range: { age: { low: 1, high: 7 } }, - }, - { string: { name: "Jan" } }, - ], + equal: { name: "Jane" }, }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], - }, - }).toFindNothing() - }) + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toFindNothing() + }) - it("can nest $and under $or filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { + !isInMemory && + it("validates conditions that are not objects", async () => { + await expect( + expectQuery({ $and: { conditions: [ - { - range: { age: { low: 1, high: 8 } }, - }, - { equal: { name: "Jan" } }, + { equal: { age: 10 } }, + "invalidCondition" as any, ], }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) - }) + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1]" must be of type object' + ) + }) - it("can nest $or under $and filters", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $or: { + !isInMemory && + it("validates $and without conditions", async () => { + await expect( + expectQuery({ + $and: { conditions: [ + { equal: { age: 10 } }, { - range: { age: { low: 1, high: 8 } }, + $and: { + conditions: undefined as any, + }, }, - { equal: { name: "Jan" } }, ], }, - equal: { name: "Jane" }, + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' + ) + }) + + it("returns no rows when onEmptyFilter set to none", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + $and: { + conditions: [{ equal: { name: "" } }], }, - ], - }, - }).toContainExactly([{ age: 1, name: "Jane" }]) - }) - - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $or: { - conditions: [{ equal: { name: "" } }], }, - }, - }).toFindNothing() - }) - - it("returns all rows when onEmptyFilter set to all", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - $or: { - conditions: [{ equal: { name: "" } }], - }, - }, - }).toHaveLength(4) - }) - }) - - isSql && - describe("max related columns", () => { - let relatedRows: Row[] - - beforeAll(async () => { - const relatedSchema: TableSchema = {} - const row: Row = {} - for (let i = 0; i < 100; i++) { - const name = `column${i}` - relatedSchema[name] = { name, type: FieldType.NUMBER } - row[name] = i - } - const relatedTable = await createTable(relatedSchema) - sourceId = await createTable({ - name: { name: "name", type: FieldType.STRING }, - related1: { - type: FieldType.LINK, - name: "related1", - fieldName: "main1", - tableId: relatedTable, - relationshipType: RelationshipType.MANY_TO_MANY, - }, + }).toFindNothing() }) - relatedRows = await Promise.all([ - config.api.row.save(relatedTable, row), - ]) - await config.api.row.save(sourceId, { - name: "foo", - related1: [relatedRows[0]._id], + + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $and: { + conditions: [{ equal: { name: "" } }], + }, + }, + }).toHaveLength(4) }) }) - it("retrieve the row with relationships", async () => { - await expectQuery({}).toContainExactly([ - { + !isLucene && + describe("$or", () => { + beforeAll(async () => { + tableOrViewId = await createSource({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, + }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Jan" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 7 } }, + }, + { string: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], + }, + }).toFindNothing() + }) + + it("can nest $and under $or filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("can nest $or under $and filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) + + it("returns no rows when onEmptyFilter set to none", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + $or: { + conditions: [{ equal: { name: "" } }], + }, + }, + }).toFindNothing() + }) + + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $or: { + conditions: [{ equal: { name: "" } }], + }, + }, + }).toHaveLength(4) + }) + }) + + isSql && + describe("max related columns", () => { + let relatedRows: Row[] + + beforeAll(async () => { + const relatedSchema: TableSchema = {} + const row: Row = {} + for (let i = 0; i < 100; i++) { + const name = `column${i}` + relatedSchema[name] = { name, type: FieldType.NUMBER } + row[name] = i + } + const relatedTable = await createSource(relatedSchema) + tableOrViewId = await createSource({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable, row), + ]) + await config.api.row.save(tableOrViewId, { name: "foo", - related1: [{ _id: relatedRows[0]._id }], - }, - ]) + related1: [relatedRows[0]._id], + }) + }) + + it("retrieve the row with relationships", async () => { + await expectQuery({}).toContainExactly([ + { + name: "foo", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) }) - }) + }) }) diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index c5614d69e7..6bec59fdf7 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -7,11 +7,11 @@ import { BulkImportRequest, BulkImportResponse, SearchRowResponse, - RowSearchParams, DeleteRows, DeleteRow, PaginatedSearchRowResponse, RowExportFormat, + SearchRowRequest, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -136,7 +136,7 @@ export class RowAPI extends TestAPI { ) } - search = async ( + search = async ( sourceId: string, params?: T, expectations?: Expectations From 470ad95208782f0c3cdd37098b81369c58551b65 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 12:37:40 +0200 Subject: [PATCH 081/105] Run view tests --- .../src/api/routes/tests/search.spec.ts | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 86ae0acf9b..6ae5490fb2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -34,6 +34,7 @@ import { Table, TableSchema, User, + ViewFieldMetadata, } from "@budibase/types" import _ from "lodash" import tk from "timekeeper" @@ -154,29 +155,33 @@ describe.each([ describe.each([ ["table", createTable], - // [ - // "view", - // async (schema: TableSchema, name?: string) => { - // const tableId = await createTable(schema, name) - // const view = await config.api.viewV2.create({ - // tableId: tableId, - // name: generator.guid(), - // schema: Object.keys(schema).reduce>( - // (viewSchema, fieldName) => { - // const field = schema[fieldName] - // viewSchema[fieldName] = { - // visible: field.visible ?? true, - // readonly: false, - // } - // return viewSchema - // }, - // {} - // ), - // }) - // return view.id - // }, - // ], - ])("from %s", (__, createSource) => { + [ + "view", + async (schema: TableSchema, name?: string) => { + const tableId = await createTable(schema, name) + const view = await config.api.viewV2.create({ + tableId: tableId, + name: generator.guid(), + schema: Object.keys(schema).reduce>( + (viewSchema, fieldName) => { + const field = schema[fieldName] + viewSchema[fieldName] = { + visible: field.visible ?? true, + readonly: false, + } + return viewSchema + }, + {} + ), + }) + return view.id + }, + ], + ])("from %s", (tableOrView, createSource) => { + if (isInMemory && tableOrView === "view") { + return + } + class SearchAssertion { constructor(private readonly query: SearchRowRequest) {} From 4b83d6b00b43d3a36133140b73f329b5a8927978 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 13:07:33 +0200 Subject: [PATCH 082/105] Fix tests --- packages/server/src/sdk/app/rows/search.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 8de5818805..2e6c14afdb 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -93,6 +93,8 @@ export async function search( // Lucene does not accept conditional filters, so we need to keep the old logic const query: SearchFilters = viewQuery + delete options.query.onEmptyFilter + // Extract existing fields const existingFields = view.query From 9586732a997036b471144e021c86d8f7abfafcda Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 13:11:15 +0200 Subject: [PATCH 083/105] Update yarn.lock --- yarn.lock | 86 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 0cddbf981f..a8f0feec45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2066,7 +2066,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.32.8": +"@budibase/backend-core@2.32.10": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2147,15 +2147,15 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "2.32.8" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.8.tgz#1b42da2ca7a496ba1ea8687cde6e7f3a8da47e48" - integrity sha512-NlY8DkD54FVcy6sL4T+wBJr/KQjXv6CGlqcHyjWVRBd8k/xLTzivrC5gVryDrkja/5ZxIm0qBKVlE81H6urLNA== + version "2.32.10" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5" + integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ== dependencies: "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "2.32.8" - "@budibase/shared-core" "2.32.8" - "@budibase/string-templates" "2.32.8" - "@budibase/types" "2.32.8" + "@budibase/backend-core" "2.32.10" + "@budibase/shared-core" "2.32.10" + "@budibase/string-templates" "2.32.10" + "@budibase/types" "2.32.10" "@koa/router" "8.0.8" bull "4.10.1" dd-trace "5.2.0" @@ -2168,13 +2168,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.32.8": +"@budibase/shared-core@2.32.10": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@2.32.8": +"@budibase/string-templates@2.32.10": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2182,7 +2182,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@2.32.8": +"@budibase/types@2.32.10": version "0.0.0" dependencies: scim-patch "^0.8.1" @@ -17758,11 +17758,21 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== + pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-connection-string@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.7.0.tgz#f1d3489e427c62ece022dba98d5262efcb168b37" + integrity sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17773,11 +17783,21 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== +pg-pool@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.7.0.tgz#d4d3c7ad640f8c6a2245adc369bafde4ebb8cbec" + integrity sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g== + pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== +pg-protocol@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.7.0.tgz#ec037c87c20515372692edac8b63cf4405448a93" + integrity sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ== + pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17802,6 +17822,19 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" +pg@^8.12.0: + version "8.13.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.13.0.tgz#e3d245342eb0158112553fcc1890a60720ae2a3d" + integrity sha512-34wkUTh3SxTClfoHB3pQ7bIMvw9dpFU1audQQeZG837fmHfHpr14n/AELVDoOYVDW2h5RDWU78tFjkD+erSBsw== + dependencies: + pg-connection-string "^2.7.0" + pg-pool "^3.7.0" + pg-protocol "^1.7.0" + pg-types "^2.1.0" + pgpass "1.x" + optionalDependencies: + pg-cloudflare "^1.1.1" + pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20696,7 +20729,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20787,7 +20829,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20801,6 +20843,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22756,7 +22805,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22774,6 +22823,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 849c7222a17d35320c20e2a99846da9e779d9d18 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 13:12:34 +0200 Subject: [PATCH 084/105] yarn dedupe --- yarn.lock | 485 +++++++----------------------------------------------- 1 file changed, 56 insertions(+), 429 deletions(-) diff --git a/yarn.lock b/yarn.lock index a8f0feec45..b95f6f4f0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17,15 +17,7 @@ resolved "https://registry.yarnpkg.com/@adobe/spectrum-css-workflow-icons/-/spectrum-css-workflow-icons-1.2.1.tgz#7e2cb3fcfb5c8b12d7275afafbb6ec44913551b4" integrity sha512-uVgekyBXnOVkxp+CUssjN/gefARtudZC8duEn1vm0lBQFwGRZFlDEzU1QC+aIRWCrD1Z8OgRpmBYlSZ7QS003w== -"@ampproject/remapping@^2.2.0", "@ampproject/remapping@^2.2.1": - version "2.2.1" - resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630" - integrity sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg== - dependencies: - "@jridgewell/gen-mapping" "^0.3.0" - "@jridgewell/trace-mapping" "^0.3.9" - -"@ampproject/remapping@^2.3.0": +"@ampproject/remapping@^2.2.0", "@ampproject/remapping@^2.2.1", "@ampproject/remapping@^2.3.0": version "2.3.0" resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.3.0.tgz#ed441b6fa600072520ce18b43d2c8cc8caecc7f4" integrity sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw== @@ -2008,14 +2000,7 @@ resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310" integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA== -"@babel/runtime@^7.10.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4": - version "7.24.7" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.7.tgz#f4f0d5530e8dbdf59b3451b9b3e594b6ba082e12" - integrity sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw== - dependencies: - regenerator-runtime "^0.14.0" - -"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.9.2": +"@babel/runtime@^7.10.5", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2": version "7.25.6" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2" integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ== @@ -2656,16 +2641,11 @@ dependencies: eslint-visitor-keys "^3.3.0" -"@eslint-community/regexpp@^4.10.0", "@eslint-community/regexpp@^4.11.0": +"@eslint-community/regexpp@^4.10.0", "@eslint-community/regexpp@^4.11.0", "@eslint-community/regexpp@^4.6.1": version "4.11.0" resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.11.0.tgz#b0ffd0312b4a3fd2d6f77237e7248a5ad3a680ae" integrity sha512-G/M/tIiMrTAxEWRfLfQJMmGNX28IxBg4PBz8XqQhqUHLFI6TL2htpIB1iQCj144V5ee/JaKyT9/WZ0MGZWfA7A== -"@eslint-community/regexpp@^4.5.1", "@eslint-community/regexpp@^4.6.1": - version "4.10.0" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.10.0.tgz#548f6de556857c8bb73bbee70c35dc82a2e74d63" - integrity sha512-Cu96Sd2By9mCNTx2iyKOmq10v22jUVQv0lQnlGNy16oE9589yE+QADPbrMGCkA51cKZSg3Pu/aTJVTGfL/qjUA== - "@eslint/config-array@^0.18.0": version "0.18.0" resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.18.0.tgz#37d8fe656e0d5e3dbaea7758ea56540867fd074d" @@ -3405,7 +3385,7 @@ dependencies: regenerator-runtime "^0.13.3" -"@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2", "@jridgewell/gen-mapping@^0.3.5": +"@jridgewell/gen-mapping@^0.3.2", "@jridgewell/gen-mapping@^0.3.5": version "0.3.5" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz#dcce6aff74bdf6dad1a95802b69b04a2fcb1fb36" integrity sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg== @@ -3432,12 +3412,7 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.25" -"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14", "@jridgewell/sourcemap-codec@^1.4.15": - version "1.4.15" - resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" - integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== - -"@jridgewell/sourcemap-codec@^1.5.0": +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14", "@jridgewell/sourcemap-codec@^1.4.15", "@jridgewell/sourcemap-codec@^1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz#3188bcb273a414b0d215fd22a58540b989b9409a" integrity sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ== @@ -3450,7 +3425,7 @@ "@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/sourcemap-codec" "^1.4.10" -"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.18", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25", "@jridgewell/trace-mapping@^0.3.9": +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.18", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.25": version "0.3.25" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== @@ -4219,161 +4194,81 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@rollup/rollup-android-arm-eabi@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz#bbd0e616b2078cd2d68afc9824d1fadb2f2ffd27" - integrity sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ== - "@rollup/rollup-android-arm-eabi@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.2.tgz#0412834dc423d1ff7be4cb1fc13a86a0cd262c11" integrity sha512-fSuPrt0ZO8uXeS+xP3b+yYTCBUd05MoSp2N/MFOgjhhUhMmchXlpTQrTpI8T+YAwAQuK7MafsCOxW7VrPMrJcg== -"@rollup/rollup-android-arm64@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz#97255ef6384c5f73f4800c0de91f5f6518e21203" - integrity sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA== - "@rollup/rollup-android-arm64@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.2.tgz#baf1a014b13654f3b9e835388df9caf8c35389cb" integrity sha512-xGU5ZQmPlsjQS6tzTTGwMsnKUtu0WVbl0hYpTPauvbRAnmIvpInhJtgjj3mcuJpEiuUw4v1s4BimkdfDWlh7gA== -"@rollup/rollup-darwin-arm64@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz#b6dd74e117510dfe94541646067b0545b42ff096" - integrity sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w== - "@rollup/rollup-darwin-arm64@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.2.tgz#0a2c364e775acdf1172fe3327662eec7c46e55b1" integrity sha512-99AhQ3/ZMxU7jw34Sq8brzXqWH/bMnf7ZVhvLk9QU2cOepbQSVTns6qoErJmSiAvU3InRqC2RRZ5ovh1KN0d0Q== -"@rollup/rollup-darwin-x64@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz#e07d76de1cec987673e7f3d48ccb8e106d42c05c" - integrity sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA== - "@rollup/rollup-darwin-x64@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.2.tgz#a972db75890dfab8df0da228c28993220a468c42" integrity sha512-ZbRaUvw2iN/y37x6dY50D8m2BnDbBjlnMPotDi/qITMJ4sIxNY33HArjikDyakhSv0+ybdUxhWxE6kTI4oX26w== -"@rollup/rollup-linux-arm-gnueabihf@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz#9f1a6d218b560c9d75185af4b8bb42f9f24736b8" - integrity sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA== - "@rollup/rollup-linux-arm-gnueabihf@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.2.tgz#1609d0630ef61109dd19a278353e5176d92e30a1" integrity sha512-ztRJJMiE8nnU1YFcdbd9BcH6bGWG1z+jP+IPW2oDUAPxPjo9dverIOyXz76m6IPA6udEL12reYeLojzW2cYL7w== -"@rollup/rollup-linux-arm-musleabihf@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz#53618b92e6ffb642c7b620e6e528446511330549" - integrity sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A== - "@rollup/rollup-linux-arm-musleabihf@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.2.tgz#3c1dca5f160aa2e79e4b20ff6395eab21804f266" integrity sha512-flOcGHDZajGKYpLV0JNc0VFH361M7rnV1ee+NTeC/BQQ1/0pllYcFmxpagltANYt8FYf9+kL6RSk80Ziwyhr7w== -"@rollup/rollup-linux-arm64-gnu@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz#99a7ba5e719d4f053761a698f7b52291cefba577" - integrity sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw== - "@rollup/rollup-linux-arm64-gnu@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.2.tgz#c2fe376e8b04eafb52a286668a8df7c761470ac7" integrity sha512-69CF19Kp3TdMopyteO/LJbWufOzqqXzkrv4L2sP8kfMaAQ6iwky7NoXTp7bD6/irKgknDKM0P9E/1l5XxVQAhw== -"@rollup/rollup-linux-arm64-musl@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz#f53db99a45d9bc00ce94db8a35efa7c3c144a58c" - integrity sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ== - "@rollup/rollup-linux-arm64-musl@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.2.tgz#e62a4235f01e0f66dbba587c087ca6db8008ec80" integrity sha512-48pD/fJkTiHAZTnZwR0VzHrao70/4MlzJrq0ZsILjLW/Ab/1XlVUStYyGt7tdyIiVSlGZbnliqmult/QGA2O2w== -"@rollup/rollup-linux-powerpc64le-gnu@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz#cbb0837408fe081ce3435cf3730e090febafc9bf" - integrity sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA== - "@rollup/rollup-linux-powerpc64le-gnu@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.2.tgz#24b3457e75ee9ae5b1c198bd39eea53222a74e54" integrity sha512-cZdyuInj0ofc7mAQpKcPR2a2iu4YM4FQfuUzCVA2u4HI95lCwzjoPtdWjdpDKyHxI0UO82bLDoOaLfpZ/wviyQ== -"@rollup/rollup-linux-riscv64-gnu@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz#8ed09c1d1262ada4c38d791a28ae0fea28b80cc9" - integrity sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg== - "@rollup/rollup-linux-riscv64-gnu@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.2.tgz#38edfba9620fe2ca8116c97e02bd9f2d606bde09" integrity sha512-RL56JMT6NwQ0lXIQmMIWr1SW28z4E4pOhRRNqwWZeXpRlykRIlEpSWdsgNWJbYBEWD84eocjSGDu/XxbYeCmwg== -"@rollup/rollup-linux-s390x-gnu@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz#938138d3c8e0c96f022252a28441dcfb17afd7ec" - integrity sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg== - "@rollup/rollup-linux-s390x-gnu@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.2.tgz#a3bfb8bc5f1e802f8c76cff4a4be2e9f9ac36a18" integrity sha512-PMxkrWS9z38bCr3rWvDFVGD6sFeZJw4iQlhrup7ReGmfn7Oukrr/zweLhYX6v2/8J6Cep9IEA/SmjXjCmSbrMQ== -"@rollup/rollup-linux-x64-gnu@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz#1a7481137a54740bee1ded4ae5752450f155d942" - integrity sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w== - "@rollup/rollup-linux-x64-gnu@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.2.tgz#0dadf34be9199fcdda44b5985a086326344f30ad" integrity sha512-B90tYAUoLhU22olrafY3JQCFLnT3NglazdwkHyxNDYF/zAxJt5fJUB/yBoWFoIQ7SQj+KLe3iL4BhOMa9fzgpw== -"@rollup/rollup-linux-x64-musl@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz#f1186afc601ac4f4fc25fac4ca15ecbee3a1874d" - integrity sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg== - "@rollup/rollup-linux-x64-musl@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.2.tgz#7b7deddce240400eb87f2406a445061b4fed99a8" integrity sha512-7twFizNXudESmC9oneLGIUmoHiiLppz/Xs5uJQ4ShvE6234K0VB1/aJYU3f/4g7PhssLGKBVCC37uRkkOi8wjg== -"@rollup/rollup-win32-arm64-msvc@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz#ed6603e93636a96203c6915be4117245c1bd2daf" - integrity sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA== - "@rollup/rollup-win32-arm64-msvc@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.2.tgz#a0ca0c5149c2cfb26fab32e6ba3f16996fbdb504" integrity sha512-9rRero0E7qTeYf6+rFh3AErTNU1VCQg2mn7CQcI44vNUWM9Ze7MSRS/9RFuSsox+vstRt97+x3sOhEey024FRQ== -"@rollup/rollup-win32-ia32-msvc@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz#14e0b404b1c25ebe6157a15edb9c46959ba74c54" - integrity sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg== - "@rollup/rollup-win32-ia32-msvc@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.2.tgz#aae2886beec3024203dbb5569db3a137bc385f8e" integrity sha512-5rA4vjlqgrpbFVVHX3qkrCo/fZTj1q0Xxpg+Z7yIo3J2AilW7t2+n6Q8Jrx+4MrYpAnjttTYF8rr7bP46BPzRw== -"@rollup/rollup-win32-x64-msvc@4.18.0": - version "4.18.0" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz#5d694d345ce36b6ecf657349e03eb87297e68da4" - integrity sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g== - "@rollup/rollup-win32-x64-msvc@4.21.2": version "4.21.2" resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.2.tgz#e4291e3c1bc637083f87936c333cdbcad22af63b" @@ -5691,7 +5586,7 @@ expect "^29.0.0" pretty-format "^29.0.0" -"@types/json-schema@*", "@types/json-schema@^7.0.12", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.9": +"@types/json-schema@*", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.9": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -5865,12 +5760,12 @@ "@types/node" "*" form-data "^4.0.0" -"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@^20.4.5": - version "20.14.5" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.5.tgz#fe35e3022ebe58b8f201580eb24e1fcfc0f2487d" - integrity sha512-aoRR+fJkZT2l0aGOJhuA8frnCSoNX6W7U2mpNq63+BxBIj5BQFt8rHy627kijCmm63ijdSdwvGgpUsU6MBsZZA== +"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=13.13.4", "@types/node@>=13.7.0", "@types/node@>=8.1.0": + version "22.5.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.1.tgz#de01dce265f6b99ed32b295962045d10b5b99560" + integrity sha512-KkHsxej0j9IW1KKOOAA/XBA0z08UFSrRQHErzEfA3Vgq57eXIMYboIlHJuYIfd+lwCQjtKqUu3UnmKbtUc9yRw== dependencies: - undici-types "~5.26.4" + undici-types "~6.19.2" "@types/node@16.9.1": version "16.9.1" @@ -5894,13 +5789,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.37.tgz#0bfcd173e8e1e328337473a8317e37b3b14fd30d" integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg== -"@types/node@>=8.1.0": - version "22.5.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.1.tgz#de01dce265f6b99ed32b295962045d10b5b99560" - integrity sha512-KkHsxej0j9IW1KKOOAA/XBA0z08UFSrRQHErzEfA3Vgq57eXIMYboIlHJuYIfd+lwCQjtKqUu3UnmKbtUc9yRw== - dependencies: - undici-types "~6.19.2" - "@types/node@^18.11.18": version "18.19.10" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.10.tgz#4de314ab66faf6bc8ba691021a091ddcdf13a158" @@ -5908,6 +5796,13 @@ dependencies: undici-types "~5.26.4" +"@types/node@^20.4.5": + version "20.14.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.5.tgz#fe35e3022ebe58b8f201580eb24e1fcfc0f2487d" + integrity sha512-aoRR+fJkZT2l0aGOJhuA8frnCSoNX6W7U2mpNq63+BxBIj5BQFt8rHy627kijCmm63ijdSdwvGgpUsU6MBsZZA== + dependencies: + undici-types "~5.26.4" + "@types/nodemailer@^6.4.4": version "6.4.15" resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.15.tgz#494be695e11c438f7f5df738fb4ab740312a6ed2" @@ -6127,12 +6022,7 @@ dependencies: "@types/retry" "*" -"@types/qs@*": - version "6.9.7" - resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" - integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== - -"@types/qs@^6.9.15": +"@types/qs@*", "@types/qs@^6.9.15": version "6.9.16" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== @@ -6219,7 +6109,7 @@ resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.7.tgz#b9eb89d7dfa70d5d1ce525bc1411a35347f533a3" integrity sha512-4g1jrL98mdOIwSOUh6LTlB0Cs9I0dQPwINUhBg7C6pN4HLr8GS8xsksJxilW6S6dQHVi2K/o+lQuQcg7LroCnw== -"@types/semver@^7.3.12", "@types/semver@^7.5.0": +"@types/semver@^7.3.12": version "7.5.8" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== @@ -6412,23 +6302,6 @@ natural-compare "^1.4.0" ts-api-utils "^1.3.0" -"@typescript-eslint/eslint-plugin@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.3.1.tgz#0d8f38a6c8a1802139e62184ee7a68ed024f30a1" - integrity sha512-STEDMVQGww5lhCuNXVSQfbfuNII5E08QWkvAw5Qwf+bj2WT+JkG1uc+5/vXA3AOYMDHVOSpL+9rcbEUiHIm2dw== - dependencies: - "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "7.3.1" - "@typescript-eslint/type-utils" "7.3.1" - "@typescript-eslint/utils" "7.3.1" - "@typescript-eslint/visitor-keys" "7.3.1" - debug "^4.3.4" - graphemer "^1.4.0" - ignore "^5.2.4" - natural-compare "^1.4.0" - semver "^7.5.4" - ts-api-utils "^1.0.1" - "@typescript-eslint/parser@6.9.0": version "6.9.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.9.0.tgz#2b402cadeadd3f211c25820e5433413347b27391" @@ -6451,17 +6324,6 @@ "@typescript-eslint/visitor-keys" "7.18.0" debug "^4.3.4" -"@typescript-eslint/parser@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.3.1.tgz#c4ba7dc2744318a5e4506596cbc3a0086255c526" - integrity sha512-Rq49+pq7viTRCH48XAbTA+wdLRrB/3sRq4Lpk0oGDm0VmnjBrAOVXH/Laalmwsv2VpekiEfVFwJYVk6/e8uvQw== - dependencies: - "@typescript-eslint/scope-manager" "7.3.1" - "@typescript-eslint/types" "7.3.1" - "@typescript-eslint/typescript-estree" "7.3.1" - "@typescript-eslint/visitor-keys" "7.3.1" - debug "^4.3.4" - "@typescript-eslint/scope-manager@5.62.0": version "5.62.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-5.62.0.tgz#d9457ccc6a0b8d6b37d0eb252a23022478c5460c" @@ -6486,14 +6348,6 @@ "@typescript-eslint/types" "7.18.0" "@typescript-eslint/visitor-keys" "7.18.0" -"@typescript-eslint/scope-manager@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.3.1.tgz#73fd0cb4211a7be23e49e5b6efec8820caa6ec36" - integrity sha512-fVS6fPxldsKY2nFvyT7IP78UO1/I2huG+AYu5AMjCT9wtl6JFiDnsv4uad4jQ0GTFzcUV5HShVeN96/17bTBag== - dependencies: - "@typescript-eslint/types" "7.3.1" - "@typescript-eslint/visitor-keys" "7.3.1" - "@typescript-eslint/type-utils@7.18.0", "@typescript-eslint/type-utils@^7.2.0": version "7.18.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.18.0.tgz#2165ffaee00b1fbbdd2d40aa85232dab6998f53b" @@ -6504,16 +6358,6 @@ debug "^4.3.4" ts-api-utils "^1.3.0" -"@typescript-eslint/type-utils@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.3.1.tgz#cbf90d3d7e788466aa8a5c0ab3f46103f098aa0d" - integrity sha512-iFhaysxFsMDQlzJn+vr3OrxN8NmdQkHks4WaqD4QBnt5hsq234wcYdyQ9uquzJJIDAj5W4wQne3yEsYA6OmXGw== - dependencies: - "@typescript-eslint/typescript-estree" "7.3.1" - "@typescript-eslint/utils" "7.3.1" - debug "^4.3.4" - ts-api-utils "^1.0.1" - "@typescript-eslint/types@4.33.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.33.0.tgz#a1e59036a3b53ae8430ceebf2a919dc7f9af6d72" @@ -6534,11 +6378,6 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.18.0.tgz#b90a57ccdea71797ffffa0321e744f379ec838c9" integrity sha512-iZqi+Ds1y4EDYUtlOOC+aUmxnE9xS/yCigkjA7XpTKV6nCBd3Hp/PRGGmdwnfkV2ThMyYldP1wRpm/id99spTQ== -"@typescript-eslint/types@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.3.1.tgz#ae104de8efa4227a462c0874d856602c5994413c" - integrity sha512-2tUf3uWggBDl4S4183nivWQ2HqceOZh1U4hhu4p1tPiIJoRRXrab7Y+Y0p+dozYwZVvLPRI6r5wKe9kToF9FIw== - "@typescript-eslint/typescript-estree@5.62.0", "@typescript-eslint/typescript-estree@^5.13.0": version "5.62.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-5.62.0.tgz#7d17794b77fabcac615d6a48fb143330d962eb9b" @@ -6579,20 +6418,6 @@ semver "^7.6.0" ts-api-utils "^1.3.0" -"@typescript-eslint/typescript-estree@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.3.1.tgz#598848195fad34c7aa73f548bd00a4d4e5f5e2bb" - integrity sha512-tLpuqM46LVkduWP7JO7yVoWshpJuJzxDOPYIVWUUZbW+4dBpgGeUdl/fQkhuV0A8eGnphYw3pp8d2EnvPOfxmQ== - dependencies: - "@typescript-eslint/types" "7.3.1" - "@typescript-eslint/visitor-keys" "7.3.1" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - minimatch "9.0.3" - semver "^7.5.4" - ts-api-utils "^1.0.1" - "@typescript-eslint/typescript-estree@^4.33.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.33.0.tgz#0dfb51c2908f68c5c08d82aefeaf166a17c24609" @@ -6616,19 +6441,6 @@ "@typescript-eslint/types" "7.18.0" "@typescript-eslint/typescript-estree" "7.18.0" -"@typescript-eslint/utils@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.3.1.tgz#fc28fd508ccf89495012561b7c02a6fdad162460" - integrity sha512-jIERm/6bYQ9HkynYlNZvXpzmXWZGhMbrOvq3jJzOSOlKXsVjrrolzWBjDW6/TvT5Q3WqaN4EkmcfdQwi9tDjBQ== - dependencies: - "@eslint-community/eslint-utils" "^4.4.0" - "@types/json-schema" "^7.0.12" - "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "7.3.1" - "@typescript-eslint/types" "7.3.1" - "@typescript-eslint/typescript-estree" "7.3.1" - semver "^7.5.4" - "@typescript-eslint/utils@^5.10.0": version "5.62.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-5.62.0.tgz#141e809c71636e4a75daa39faed2fb5f4b10df86" @@ -6675,14 +6487,6 @@ "@typescript-eslint/types" "7.18.0" eslint-visitor-keys "^3.4.3" -"@typescript-eslint/visitor-keys@7.3.1": - version "7.3.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.3.1.tgz#6ddef14a3ce2a79690f01176f5305c34d7b93d8c" - integrity sha512-9RMXwQF8knsZvfv9tdi+4D/j7dMG28X/wMJ8Jj6eOHyHWwDW4ngQJcqEczSsqIKKjFiLFr40Mnr7a5ulDD3vmw== - dependencies: - "@typescript-eslint/types" "7.3.1" - eslint-visitor-keys "^3.4.1" - "@ungap/structured-clone@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" @@ -6943,16 +6747,11 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.1.0, acorn@^8.11.3, acorn@^8.12.0, acorn@^8.12.1, acorn@^8.8.1: +acorn@^8.1.0, acorn@^8.10.0, acorn@^8.11.0, acorn@^8.11.3, acorn@^8.12.0, acorn@^8.12.1, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0: version "8.12.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== -acorn@^8.10.0, acorn@^8.11.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.2, acorn@^8.9.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" - integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== - add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -7015,17 +6814,7 @@ ajv@^6.12.3, ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.1.0, ajv@^8.4.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" - integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== - dependencies: - fast-deep-equal "^3.1.1" - json-schema-traverse "^1.0.0" - require-from-string "^2.0.2" - uri-js "^4.2.2" - -ajv@^8.17.1: +ajv@^8.0.0, ajv@^8.1.0, ajv@^8.17.1, ajv@^8.4.0: version "8.17.1" resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.17.1.tgz#37d9a5c776af6bc92d7f4f9510eba4c0a60d11a6" integrity sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g== @@ -7402,7 +7191,7 @@ asn1.js@^4.10.1: inherits "^2.0.1" minimalistic-assert "^1.0.0" -asn1.js@^5.0.0, asn1.js@^5.2.0, asn1.js@^5.4.1: +asn1.js@^5.0.0, asn1.js@^5.4.1: version "5.4.1" resolved "https://registry.yarnpkg.com/asn1.js/-/asn1.js-5.4.1.tgz#11a980b84ebb91781ce35b0fdc2ee294e3783f07" integrity sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA== @@ -7913,7 +7702,7 @@ browser-request@^0.3.3: resolved "https://registry.yarnpkg.com/browser-request/-/browser-request-0.3.3.tgz#9ece5b5aca89a29932242e18bf933def9876cc17" integrity sha512-YyNI4qJJ+piQG6MMEuo7J3Bzaqssufx04zpEKYfSrl/1Op59HWali9zMtBpXnkmqMcOuWJPZvudrm9wISmnCbg== -browserify-aes@^1.0.0, browserify-aes@^1.0.4, browserify-aes@^1.2.0: +browserify-aes@^1.0.4, browserify-aes@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/browserify-aes/-/browserify-aes-1.2.0.tgz#326734642f403dabc3003209853bb70ad428ef48" integrity sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA== @@ -9160,14 +8949,7 @@ crelt@^1.0.5: resolved "https://registry.yarnpkg.com/crelt/-/crelt-1.0.5.tgz#57c0d52af8c859e354bace1883eb2e1eb182bb94" integrity sha512-+BO9wPPi+DWTDcNYhr/W90myha8ptzftZT+LwcmUbbok0rcP/fequmFYCw8NMoH7pkAZQzU78b3kYrlua5a9eA== -cron-parser@^4.2.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.7.1.tgz#1e325a6a18e797a634ada1e2599ece0b6b5ed177" - integrity sha512-WguFaoQ0hQ61SgsCZLHUcNbAvlK0lypKXu62ARguefYmjzaOXIVRNrAmyXzabTwUn4sQvQLkk6bjH+ipGfw8bA== - dependencies: - luxon "^3.2.1" - -cron-parser@^4.9.0: +cron-parser@^4.2.1, cron-parser@^4.9.0: version "4.9.0" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5" integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q== @@ -9444,12 +9226,7 @@ dateformat@^4.6.3: resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5" integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA== -dayjs@^1.10.8: - version "1.11.11" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e" - integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg== - -dayjs@^1.8.15: +dayjs@^1.10.8, dayjs@^1.8.15: version "1.11.13" resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.13.tgz#92430b0139055c3ebb60150aa13e860a4b5a366c" integrity sha512-oaMBel6gjolK862uaPQOVTA7q3TZhuSvuMQAAglQDOWYO9A91IrAOUJEyKVlqJlHE0vq5p5UXxzdPfMH/x6xNg== @@ -9496,10 +9273,10 @@ dd-trace@5.2.0: semver "^7.5.4" tlhunter-sorted-set "^0.1.0" -debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2, debug@~4.3.4: - version "4.3.5" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.5.tgz#e83444eceb9fedd4a1da56d671ae2446a01a6e1e" - integrity sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg== +debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@^4.3.5, debug@~4.3.1, debug@~4.3.2, debug@~4.3.4: + version "4.3.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" + integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg== dependencies: ms "2.1.2" @@ -9517,13 +9294,6 @@ debug@^3.1.0, debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.3.5: - version "4.3.6" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" - integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg== - dependencies: - ms "2.1.2" - debuglog@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" @@ -10475,7 +10245,7 @@ engine.io@~6.5.2: engine.io-parser "~5.2.1" ws "~8.17.1" -enhanced-resolve@^5.17.1, enhanced-resolve@^5.7.0: +enhanced-resolve@^5.17.1, enhanced-resolve@^5.7.0, enhanced-resolve@^5.8.3: version "5.17.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz#67bfbbcc2f81d511be77d686a90267ef7f898a15" integrity sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg== @@ -10483,14 +10253,6 @@ enhanced-resolve@^5.17.1, enhanced-resolve@^5.7.0: graceful-fs "^4.2.4" tapable "^2.2.0" -enhanced-resolve@^5.8.3: - version "5.14.1" - resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.14.1.tgz#de684b6803724477a4af5d74ccae5de52c25f6b3" - integrity sha512-Vklwq2vDKtl0y/vtwjSesgJ5MYS7Etuk5txS8VdKL4AOS1aUlD96zqIfsOSLQsdv3xgMRbtkWM8eG9XDfKUPow== - dependencies: - graceful-fs "^4.2.4" - tapable "^2.2.0" - enquirer@~2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" @@ -11060,14 +10822,7 @@ esprima@~3.1.0: resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" integrity sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg== -esquery@^1.4.2: - version "1.5.0" - resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.5.0.tgz#6ce17738de8577694edd7361c57182ac8cb0db0b" - integrity sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg== - dependencies: - estraverse "^5.1.0" - -esquery@^1.5.0: +esquery@^1.4.2, esquery@^1.5.0: version "1.6.0" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.6.0.tgz#91419234f804d852a82dceec3e16cdc22cf9dae7" integrity sha512-ca9pw9fomFcKPvFLXhBKUK90ZvGibiGOvRJNbjljY7s7uq/5YO4BOzcYtJqExdx99rF6aAcnRxHmcUHcz6sQsg== @@ -11605,12 +11360,7 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== -flatted@^3.1.0: - version "3.2.5" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" - integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== - -flatted@^3.2.9: +flatted@^3.1.0, flatted@^3.2.9: version "3.3.1" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== @@ -12152,18 +11902,7 @@ glob@7.1.6: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^10.0.0, glob@^10.2.2: - version "10.4.1" - resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.1.tgz#0cfb01ab6a6b438177bfe6a58e2576f6efe909c2" - integrity sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw== - dependencies: - foreground-child "^3.1.0" - jackspeak "^3.1.2" - minimatch "^9.0.4" - minipass "^7.1.2" - path-scurry "^1.11.1" - -glob@^10.3.7: +glob@^10.0.0, glob@^10.2.2, glob@^10.3.7: version "10.4.5" resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== @@ -12912,12 +12651,7 @@ ignore-walk@^6.0.0: dependencies: minimatch "^7.4.2" -ignore@^5.0.4, ignore@^5.2.0, ignore@^5.2.4: - version "5.3.0" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78" - integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg== - -ignore@^5.3.1, ignore@^5.3.2: +ignore@^5.0.4, ignore@^5.2.0, ignore@^5.2.4, ignore@^5.3.1, ignore@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== @@ -15636,20 +15370,13 @@ magic-string@^0.26.7: dependencies: sourcemap-codec "^1.4.8" -magic-string@^0.30.10: +magic-string@^0.30.10, magic-string@^0.30.3, magic-string@^0.30.4: version "0.30.11" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.11.tgz#301a6f93b3e8c2cb13ac1a7a673492c0dfd12954" integrity sha512-+Wri9p0QHMy+545hKww7YAu5NyzF8iomPL/RQazugQ9+Ez4Ic3mERMd8ZTX5rfK944j+560ZJi8iAwgak1Ac7A== dependencies: "@jridgewell/sourcemap-codec" "^1.5.0" -magic-string@^0.30.3, magic-string@^0.30.4: - version "0.30.10" - resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.10.tgz#123d9c41a0cb5640c892b041d4cfb3bd0aa4b39e" - integrity sha512-iIRwTIf0QKV3UAnYK4PU8uiEc4SRh5jX0mwpIwETPpHdhVM4f53RSwS/vXvN1JhGX+Cs7B8qIq3d6AH49O5fAQ== - dependencies: - "@jridgewell/sourcemap-codec" "^1.4.15" - make-dir@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-4.0.0.tgz#c3c2307a771277cd9638305f915c29ae741b614e" @@ -15991,13 +15718,6 @@ minimatch@3.0.5: dependencies: brace-expansion "^1.1.7" -minimatch@9.0.3: - version "9.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" - integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== - dependencies: - brace-expansion "^2.0.1" - minimatch@^10.0.0: version "10.0.1" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.0.1.tgz#ce0521856b453c86e25f2c4c0d03e6ff7ddc440b" @@ -16810,12 +16530,7 @@ nunjucks@^3.2.3: asap "^2.0.3" commander "^5.1.0" -nwsapi@^2.2.0: - version "2.2.4" - resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.4.tgz#fd59d5e904e8e1f03c25a7d5a15cfa16c714a1e5" - integrity sha512-NHj4rzRo0tQdijE9ZqAx6kYDcoRwYwSYzCA8MY3JzfxlrvEU0jhnhJT9BhqhJs7I/dKcrDm6TyulaRqZPIhN5g== - -nwsapi@^2.2.4: +nwsapi@^2.2.0, nwsapi@^2.2.4: version "2.2.12" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.12.tgz#fb6af5c0ec35b27b4581eb3bbad34ec9e5c696f8" integrity sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w== @@ -17426,18 +17141,7 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" -parse-asn1@^5.0.0: - version "5.1.6" - resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.6.tgz#385080a3ec13cb62a62d39409cb3e88844cdaed4" - integrity sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw== - dependencies: - asn1.js "^5.2.0" - browserify-aes "^1.0.0" - evp_bytestokey "^1.0.0" - pbkdf2 "^3.0.3" - safe-buffer "^5.1.1" - -parse-asn1@^5.1.7: +parse-asn1@^5.0.0, parse-asn1@^5.1.7: version "5.1.7" resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.7.tgz#73cdaaa822125f9647165625eb45f8a051d2df06" integrity sha512-CTM5kuWR3sx9IFamcl5ErfPl6ea/N8IYwiJ+vpeB2g+1iknv7zBl5uPwbMbRVznRVbrNY6lGuDoE5b30grmbqg== @@ -17763,12 +17467,12 @@ pg-cloudflare@^1.1.1: resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== -pg-connection-string@2.5.0, pg-connection-string@^2.5.0: +pg-connection-string@2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== -pg-connection-string@^2.7.0: +pg-connection-string@^2.5.0, pg-connection-string@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.7.0.tgz#f1d3489e427c62ece022dba98d5262efcb168b37" integrity sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA== @@ -17778,22 +17482,12 @@ pg-int8@1.0.1: resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== -pg-pool@^3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" - integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== - -pg-pool@^3.7.0: +pg-pool@^3.6.0, pg-pool@^3.7.0: version "3.7.0" resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.7.0.tgz#d4d3c7ad640f8c6a2245adc369bafde4ebb8cbec" integrity sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g== -pg-protocol@*, pg-protocol@^1.6.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" - integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== - -pg-protocol@^1.7.0: +pg-protocol@*, pg-protocol@^1.6.0, pg-protocol@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.7.0.tgz#ec037c87c20515372692edac8b63cf4405448a93" integrity sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ== @@ -17847,12 +17541,7 @@ phin@^2.9.1: resolved "https://registry.yarnpkg.com/phin/-/phin-2.9.3.tgz#f9b6ac10a035636fb65dfc576aaaa17b8743125c" integrity sha512-CzFr90qM24ju5f88quFC/6qohjC144rehe5n6DH900lgXmUe86+xCKc10ev56gRKC4/BkHUoG4uSiQgBiIXwDA== -picocolors@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" - integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ== - -picocolors@^1.0.1: +picocolors@^1.0.0, picocolors@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew== @@ -18345,16 +18034,7 @@ postcss-values-parser@^6.0.2: is-url-superb "^4.0.0" quote-unquote "^1.0.0" -postcss@^8.1.7, postcss@^8.2.9, postcss@^8.3.11, postcss@^8.4.12, postcss@^8.4.27, postcss@^8.4.29, postcss@^8.4.35, postcss@^8.4.5: - version "8.4.38" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.38.tgz#b387d533baf2054288e337066d81c6bee9db9e0e" - integrity sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A== - dependencies: - nanoid "^3.3.7" - picocolors "^1.0.0" - source-map-js "^1.2.0" - -postcss@^8.4.41: +postcss@^8.1.7, postcss@^8.2.9, postcss@^8.3.11, postcss@^8.4.12, postcss@^8.4.27, postcss@^8.4.29, postcss@^8.4.35, postcss@^8.4.41, postcss@^8.4.5: version "8.4.41" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.41.tgz#d6104d3ba272d882fe18fc07d15dc2da62fa2681" integrity sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ== @@ -18385,15 +18065,7 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" -posthog-js@^1.118.0: - version "1.139.2" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.139.2.tgz#f8de29edf2770da47fcccb7838902d1e89d6b43d" - integrity sha512-myyuOADqZvYwgqmriwlKDEUDwLhscivFLh67UWBj4Wt9kOlmklvJb36W0ES2GAS6IdojbnGZGH5lF3heqreLWQ== - dependencies: - fflate "^0.4.8" - preact "^10.19.3" - -posthog-js@^1.13.4: +posthog-js@^1.118.0, posthog-js@^1.13.4: version "1.160.0" resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.160.0.tgz#ad686f3c161c7dc2ba716281b5cef94c64ce41b1" integrity sha512-K/RRgmPYIpP69nnveCJfkclb8VU+R+jsgqlrKaLGsM5CtQM9g01WOzAiT3u36WLswi58JiFMXgJtECKQuoqTgQ== @@ -18975,20 +18647,13 @@ q@^1.1.2: resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw== -qs@^6.10.3: +qs@^6.10.3, qs@^6.11.0, qs@^6.4.0: version "6.13.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: side-channel "^1.0.6" -qs@^6.11.0, qs@^6.4.0: - version "6.12.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.1.tgz#39422111ca7cbdb70425541cba20c7d7b216599a" - integrity sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ== - dependencies: - side-channel "^1.0.6" - qs@~6.5.2: version "6.5.3" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" @@ -19802,7 +19467,7 @@ rollup@^3.27.1: optionalDependencies: fsevents "~2.3.2" -rollup@^4.20.0: +rollup@^4.20.0, rollup@^4.9.4, rollup@^4.9.6: version "4.21.2" resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.21.2.tgz#f41f277a448d6264e923dd1ea179f0a926aaf9b7" integrity sha512-e3TapAgYf9xjdLvKQCkQTnbTKd4a6jwlpQSJJFokHGaX2IVjoEqkIIhiQfqsi0cdwlOD+tQGuOd5AJkc5RngBw== @@ -19827,31 +19492,6 @@ rollup@^4.20.0: "@rollup/rollup-win32-x64-msvc" "4.21.2" fsevents "~2.3.2" -rollup@^4.9.4, rollup@^4.9.6: - version "4.18.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.18.0.tgz#497f60f0c5308e4602cf41136339fbf87d5f5dda" - integrity sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg== - dependencies: - "@types/estree" "1.0.5" - optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.18.0" - "@rollup/rollup-android-arm64" "4.18.0" - "@rollup/rollup-darwin-arm64" "4.18.0" - "@rollup/rollup-darwin-x64" "4.18.0" - "@rollup/rollup-linux-arm-gnueabihf" "4.18.0" - "@rollup/rollup-linux-arm-musleabihf" "4.18.0" - "@rollup/rollup-linux-arm64-gnu" "4.18.0" - "@rollup/rollup-linux-arm64-musl" "4.18.0" - "@rollup/rollup-linux-powerpc64le-gnu" "4.18.0" - "@rollup/rollup-linux-riscv64-gnu" "4.18.0" - "@rollup/rollup-linux-s390x-gnu" "4.18.0" - "@rollup/rollup-linux-x64-gnu" "4.18.0" - "@rollup/rollup-linux-x64-musl" "4.18.0" - "@rollup/rollup-win32-arm64-msvc" "4.18.0" - "@rollup/rollup-win32-ia32-msvc" "4.18.0" - "@rollup/rollup-win32-x64-msvc" "4.18.0" - fsevents "~2.3.2" - rotating-file-stream@3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/rotating-file-stream/-/rotating-file-stream-3.1.0.tgz#6cf50e1671de82a396de6d31d39a6f2445f45fba" @@ -21676,12 +21316,7 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== -ts-api-utils@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.3.tgz#f12c1c781d04427313dbac808f453f050e54a331" - integrity sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg== - -ts-api-utils@^1.3.0: +ts-api-utils@^1.0.1, ts-api-utils@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1" integrity sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ== @@ -21985,7 +21620,7 @@ typeof@^1.0.0: resolved "https://registry.yarnpkg.com/typeof/-/typeof-1.0.0.tgz#9c84403f2323ae5399167275497638ea1d2f2440" integrity sha512-Pze0mIxYXhaJdpw1ayMzOA7rtGr1OmsTY/Z+FWtRKIqXFz6aoDLjqdbWE/tcIBSC8nhnVXiRrEXujodR/xiFAA== -typescript-eslint@^7.16.1: +typescript-eslint@^7.16.1, typescript-eslint@^7.3.1: version "7.18.0" resolved "https://registry.yarnpkg.com/typescript-eslint/-/typescript-eslint-7.18.0.tgz#e90d57649b2ad37a7475875fa3e834a6d9f61eb2" integrity sha512-PonBkP603E3tt05lDkbOMyaxJjvKqQrXsnow72sVeOFINDE/qNmnnd+f9b4N+U7W6MXnnYyrhtmF2t08QWwUbA== @@ -21994,19 +21629,16 @@ typescript-eslint@^7.16.1: "@typescript-eslint/parser" "7.18.0" "@typescript-eslint/utils" "7.18.0" -typescript-eslint@^7.3.1: - version "7.3.1" - resolved "https://registry.yarnpkg.com/typescript-eslint/-/typescript-eslint-7.3.1.tgz#9f4808abea3b33c4dd3bb51dd801471e91d1bd58" - integrity sha512-psqcnHPRCdVIDbgj6RvfpwUKqMcNxIw7eizgxYi46X2BmXK6LxYqPD+SbDfPuA9JW+yPItY6aKJLRNbW7lZ4rA== - dependencies: - "@typescript-eslint/eslint-plugin" "7.3.1" - "@typescript-eslint/parser" "7.3.1" - -typescript@5.5.2, "typescript@>=3 < 6": +typescript@5.5.2: version "5.5.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507" integrity sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew== +"typescript@>=3 < 6", typescript@^5.5.3: + version "5.5.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.4.tgz#d9852d6c82bad2d2eda4fd74a5762a8f5909e9ba" + integrity sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q== + typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7: version "3.9.10" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8" @@ -22017,11 +21649,6 @@ typescript@^4.0.0, typescript@^4.5.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== -typescript@^5.5.3: - version "5.5.4" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.4.tgz#d9852d6c82bad2d2eda4fd74a5762a8f5909e9ba" - integrity sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q== - typo-js@*: version "1.2.2" resolved "https://registry.yarnpkg.com/typo-js/-/typo-js-1.2.2.tgz#340484d81fe518e77c81a5a770162b14492f183b" From bc96e61baf170601a304e9a64c585c04ab605ce7 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 2 Oct 2024 12:57:10 +0100 Subject: [PATCH 085/105] Avoid mutating path array in-place --- packages/client/src/components/app/forms/Form.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte index 5522bd4b46..fe02dc665d 100644 --- a/packages/client/src/components/app/forms/Form.svelte +++ b/packages/client/src/components/app/forms/Form.svelte @@ -63,7 +63,7 @@ // Look up the component tree and find something that is provided by an // ancestor that matches our datasource. This is for backwards compatibility // as previously we could use the "closest" context. - for (let id of path.reverse().slice(1)) { + for (let id of path.toReversed().slice(1)) { // Check for matching view datasource if ( dataSource.type === "viewV2" && From 0d64e03c9f0a523c96f0f03223fa0ef7218accdc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 14:10:29 +0200 Subject: [PATCH 086/105] Don't run for lucene views --- packages/server/src/api/routes/tests/search.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 6ae5490fb2..e9f80b0b2e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -178,7 +178,8 @@ describe.each([ }, ], ])("from %s", (tableOrView, createSource) => { - if (isInMemory && tableOrView === "view") { + if (tableOrView === "view" && isLucene) { + // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests return } From 45a6f0680f2c2dd4d14d82e019f0dd7390b026f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 13:58:23 +0100 Subject: [PATCH 087/105] Fix view.spec.ts tests. --- packages/server/src/sdk/app/views/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index d218a3c7e8..2bd90822c7 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -293,14 +293,16 @@ export async function enrichSchema( const viewSchema = view.schema || {} const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null) - const visibleSchemaFields = Object.keys(viewSchema).filter( - key => viewSchema[key].visible !== false - ) + const visibleSchemaFields = Object.keys(viewSchema).filter(key => { + if (helpers.views.isCalculationField(viewSchema[key])) { + return viewSchema[key].visible !== false + } + return key in tableSchema && tableSchema[key].visible !== false + }) const visibleTableFields = Object.keys(tableSchema).filter( key => tableSchema[key].visible !== false ) const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields]) - for (const key of visibleFields) { // if nothing specified in view, then it is not visible const ui = viewSchema[key] || { visible: false } From 6d8cb1d1aa7137cb7f097274ed99387eab4355d1 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Wed, 2 Oct 2024 17:02:19 +0300 Subject: [PATCH 088/105] Adapt the payloads for the change in the deployment --- .github/workflows/deploy-featurebranch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index d86d301507..a676fe75f0 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -23,6 +23,7 @@ jobs: PAYLOAD_BRANCH: ${{ github.head_ref }} PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} PAYLOAD_LICENSE_TYPE: "free" + PAYLOAD_DEPLOY: "true" with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy From c4a98832cbc6ee8da82ade1ad64eee85e85ffd4b Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 2 Oct 2024 15:24:04 +0100 Subject: [PATCH 089/105] Enable default values for options and array types --- packages/server/src/utilities/rowProcessor/index.ts | 6 ++++-- packages/shared-core/src/table.ts | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 7332f8b244..e63750bff9 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -134,8 +134,10 @@ async function processDefaultValues(table: Table, row: Row) { for (const [key, schema] of Object.entries(table.schema)) { if ("default" in schema && schema.default != null && row[key] == null) { - const processed = await processString(schema.default, ctx) - + const processed = + typeof schema.default === "string" + ? await processString(schema.default, ctx) + : schema.default try { row[key] = coerce(processed, schema.type) } catch (err: any) { diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 8a8069ce4d..57f6854604 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -53,8 +53,9 @@ const allowDefaultColumnByType: Record = { [FieldType.DATETIME]: true, [FieldType.LONGFORM]: true, [FieldType.STRING]: true, + [FieldType.OPTIONS]: true, + [FieldType.ARRAY]: true, - [FieldType.OPTIONS]: false, [FieldType.AUTO]: false, [FieldType.INTERNAL]: false, [FieldType.BARCODEQR]: false, @@ -64,7 +65,6 @@ const allowDefaultColumnByType: Record = { [FieldType.ATTACHMENTS]: false, [FieldType.ATTACHMENT_SINGLE]: false, [FieldType.SIGNATURE_SINGLE]: false, - [FieldType.ARRAY]: false, [FieldType.LINK]: false, [FieldType.BB_REFERENCE]: false, [FieldType.BB_REFERENCE_SINGLE]: false, From 3e3fd20207cb0139f403e25c3b4a5c977c2b19f4 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 2 Oct 2024 15:35:17 +0100 Subject: [PATCH 090/105] Add tests for new default value types --- .../server/src/api/routes/tests/row.spec.ts | 63 +++++++++++++++++++ .../types/src/documents/app/table/schema.ts | 2 + 2 files changed, 65 insertions(+) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9ccf919ff8..f751942df9 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -695,6 +695,69 @@ describe.each([ }) }) + describe("options column", () => { + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + status: { + name: "status", + type: FieldType.OPTIONS, + default: "requested", + constraints: { + inclusion: ["requested", "approved"], + }, + }, + }, + }) + ) + }) + + it("creates a new row with a default value successfully", async () => { + const row = await config.api.row.save(table._id!, {}) + expect(row.status).toEqual("requested") + }) + + it("does not use default value if value specified", async () => { + const row = await config.api.row.save(table._id!, { + status: "approved", + }) + expect(row.status).toEqual("approved") + }) + }) + + describe("array column", () => { + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + food: { + name: "food", + type: FieldType.ARRAY, + default: ["apple", "orange"], + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["apple", "orange", "banana"], + }, + }, + }, + }) + ) + }) + + it("creates a new row with a default value successfully", async () => { + const row = await config.api.row.save(table._id!, {}) + expect(row.food).toEqual(["apple", "orange"]) + }) + + it("does not use default value if value specified", async () => { + const row = await config.api.row.save(table._id!, { + food: ["orange"], + }) + expect(row.food).toEqual(["orange"]) + }) + }) + describe("bindings", () => { describe("string column", () => { beforeAll(async () => { diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 6078f73d1d..f9d1a4c012 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -161,6 +161,7 @@ export interface OptionsFieldMetadata extends BaseFieldSchema { constraints: FieldConstraints & { inclusion: string[] } + default?: string } export interface ArrayFieldMetadata extends BaseFieldSchema { @@ -169,6 +170,7 @@ export interface ArrayFieldMetadata extends BaseFieldSchema { type: JsonFieldSubType.ARRAY inclusion: string[] } + default?: string[] } interface BaseFieldSchema extends UIFieldMetadata { From 557add173a6a35d422e61051633e35f5343a7b8c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 14:18:59 +0200 Subject: [PATCH 091/105] Fix view relationships --- .../src/api/routes/tests/search.spec.ts | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index e9f80b0b2e..b69631a006 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -177,7 +177,7 @@ describe.each([ return view.id }, ], - ])("from %s", (tableOrView, createSource) => { + ])("from %s", (tableOrView, createTableOrView) => { if (tableOrView === "view" && isLucene) { // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests return @@ -373,7 +373,7 @@ describe.each([ describe("boolean", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, }) await createRows([{ isTrue: true }, { isTrue: false }]) @@ -511,7 +511,7 @@ describe.each([ }) ) - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING }, appointment: { name: "appointment", type: FieldType.DATETIME }, single_user: { @@ -797,7 +797,7 @@ describe.each([ describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING }, }) await createRows([{ name: "foo" }, { name: "bar" }]) @@ -1112,7 +1112,7 @@ describe.each([ describe("numbers", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ age: { name: "age", type: FieldType.NUMBER }, }) await createRows([{ age: 1 }, { age: 10 }]) @@ -1287,7 +1287,7 @@ describe.each([ const JAN_10TH = "2020-01-10T00:00:00.000Z" beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ dob: { name: "dob", type: FieldType.DATETIME }, }) @@ -1442,7 +1442,7 @@ describe.each([ const NULL_TIME__ID = `null_time__id` beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ timeid: { name: "timeid", type: FieldType.STRING }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, }) @@ -1603,7 +1603,7 @@ describe.each([ describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ numbers: { name: "numbers", type: FieldType.ARRAY, @@ -1700,7 +1700,7 @@ describe.each([ let BIG = "9223372036854775807" beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ num: { name: "num", type: FieldType.BIGINT }, }) await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) @@ -1805,7 +1805,7 @@ describe.each([ isInternal && describe("auto", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ auto: { name: "auto", type: FieldType.AUTO, @@ -2016,7 +2016,7 @@ describe.each([ describe("field name 1:name", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ "1:name": { name: "1:name", type: FieldType.STRING }, }) await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) @@ -2036,7 +2036,7 @@ describe.each([ isSql && describe("related formulas", () => { beforeAll(async () => { - const arrayTable = await createSource( + const arrayTable = await createTable( { name: { name: "name", type: FieldType.STRING }, array: { @@ -2050,7 +2050,7 @@ describe.each([ }, "array" ) - tableOrViewId = await createSource( + tableOrViewId = await createTableOrView( { relationship: { type: FieldType.LINK, @@ -2102,7 +2102,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ user: { name: "user", type: FieldType.BB_REFERENCE_SINGLE, @@ -2175,7 +2175,7 @@ describe.each([ user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` }) - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ users: { name: "users", type: FieldType.BB_REFERENCE, @@ -2372,13 +2372,13 @@ describe.each([ let relatedTable: string, relatedRows: Row[] beforeAll(async () => { - relatedTable = await createSource( + relatedTable = await createTable( { name: { name: "name", type: FieldType.STRING }, }, "productCategory" ) - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, @@ -2470,7 +2470,7 @@ describe.each([ isInternal && describe("no column error backwards compat", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING, @@ -2493,7 +2493,7 @@ describe.each([ !isLucene && describe("row counting", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING, @@ -2528,7 +2528,7 @@ describe.each([ describe("Invalid column definitions", () => { beforeAll(async () => { // need to create an invalid table - means ignoring typescript - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ // @ts-ignore invalid: { type: FieldType.STRING, @@ -2558,7 +2558,7 @@ describe.each([ "special (%s) case", column => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ [column]: { name: column, type: FieldType.STRING, @@ -2607,7 +2607,7 @@ describe.each([ const earlyDate = "2024-07-03T10:00:00.000Z", laterDate = "2024-07-03T11:00:00.000Z" beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ date: { name: "date", type: FieldType.DATETIME, @@ -2650,7 +2650,7 @@ describe.each([ "ชื่อผู้ใช้", // Thai for "username" ])("non-ascii column name: %s", name => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ [name]: { name, type: FieldType.STRING, @@ -2677,7 +2677,7 @@ describe.each([ isInternal && describe("space at end of column name", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ "name ": { name: "name ", type: FieldType.STRING, @@ -2712,7 +2712,7 @@ describe.each([ ;(isSqs || isInMemory) && describe("space at start of column name", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ " name": { name: " name", type: FieldType.STRING, @@ -2745,7 +2745,7 @@ describe.each([ isSqs && describe("duplicate columns", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING, @@ -2781,13 +2781,13 @@ describe.each([ let row: Row beforeAll(async () => { - const toRelateTable = await createSource({ + const toRelateTable = await createTable({ name: { name: "name", type: FieldType.STRING, }, }) - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING, @@ -2860,7 +2860,7 @@ describe.each([ isSql && describe("primaryDisplay", () => { beforeAll(async () => { - let toRelateTableId = await createSource({ + let toRelateTableId = await createTableOrView({ name: { name: "name", type: FieldType.STRING, @@ -2912,7 +2912,7 @@ describe.each([ !isLucene && describe("$and", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -3044,7 +3044,7 @@ describe.each([ !isLucene && describe("$or", () => { beforeAll(async () => { - tableOrViewId = await createSource({ + tableOrViewId = await createTableOrView({ age: { name: "age", type: FieldType.NUMBER }, name: { name: "name", type: FieldType.STRING }, }) @@ -3203,8 +3203,8 @@ describe.each([ relatedSchema[name] = { name, type: FieldType.NUMBER } row[name] = i } - const relatedTable = await createSource(relatedSchema) - tableOrViewId = await createSource({ + const relatedTable = await createTable(relatedSchema) + tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING }, related1: { type: FieldType.LINK, From 0484b798fc58b4a1bc8621adc022c48460e1a144 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 16:22:17 +0200 Subject: [PATCH 092/105] Honor onEmptyFilter in views --- packages/server/src/sdk/app/rows/search.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 2e6c14afdb..fd003ff56a 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -113,6 +113,7 @@ export async function search( options.query = query } else { options.query = { + onEmptyFilter: viewQuery.onEmptyFilter, $and: { conditions: [viewQuery, options.query], }, From 521a14650d264564212b54c9691816aadc700e14 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 16:50:50 +0200 Subject: [PATCH 093/105] Fix tests --- .../src/api/routes/tests/search.spec.ts | 119 +++++++++--------- packages/server/src/sdk/app/rows/search.ts | 4 +- 2 files changed, 65 insertions(+), 58 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index b69631a006..679ff0e356 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -178,7 +178,9 @@ describe.each([ }, ], ])("from %s", (tableOrView, createTableOrView) => { - if (tableOrView === "view" && isLucene) { + const isView = tableOrView === "view" + + if (isView && isLucene) { // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests return } @@ -824,11 +826,13 @@ describe.each([ }).toContainExactly([{ name: "foo" }, { name: "bar" }]) }) - it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toFindNothing() - }) + // onEmptyFilter cannot be sent to view searches + !isView && + it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { + await expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }).toFindNothing() + }) it("should respect limit", async () => { await expectSearch({ @@ -924,12 +928,14 @@ describe.each([ }).toContainExactly([{ name: "foo" }, { name: "bar" }]) }) - it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { - await expectQuery({ - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - oneOf: { name: [] }, - }).toContainExactly([]) - }) + // onEmptyFilter cannot be sent to view searches + !isView && + it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { + await expectQuery({ + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + oneOf: { name: [] }, + }).toContainExactly([]) + }) }) describe("fuzzy", () => { @@ -2743,6 +2749,7 @@ describe.each([ }) isSqs && + !isView && describe("duplicate columns", () => { beforeAll(async () => { tableOrViewId = await createTableOrView({ @@ -2860,51 +2867,45 @@ describe.each([ isSql && describe("primaryDisplay", () => { beforeAll(async () => { - let toRelateTableId = await createTableOrView({ + let toRelateTableId = await createTable({ name: { name: "name", type: FieldType.STRING, }, }) - const table = await config.api.table.save( - tableForDatasource(datasource, { - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - link: { - name: "link", - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - tableId: toRelateTableId, - fieldName: "link", - }, - }, - }) - ) - tableOrViewId = table._id! + tableOrViewId = await createTableOrView({ + name: { + name: "name", + type: FieldType.STRING, + }, + link: { + name: "link", + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_ONE, + tableId: toRelateTableId, + fieldName: "link", + }, + }) + const toRelateTable = await config.api.table.get(toRelateTableId) await config.api.table.save({ ...toRelateTable, primaryDisplay: "link", }) const relatedRows = await Promise.all([ - config.api.row.save(toRelateTable._id!, { name: "test" }), - ]) - await Promise.all([ - config.api.row.save(tableOrViewId, { - name: "test", - link: relatedRows.map(row => row._id), - }), + config.api.row.save(toRelateTable._id!, { name: "related" }), ]) + await config.api.row.save(tableOrViewId, { + name: "test", + link: relatedRows.map(row => row._id), + }) }) it("should be able to query, primary display on related table shouldn't be used", async () => { // this test makes sure that if a relationship has been specified as the primary display on a table // it is ignored and another column is used instead await expectQuery({}).toContain([ - { name: "test", link: [{ primaryDisplay: "test" }] }, + { name: "test", link: [{ primaryDisplay: "related" }] }, ]) }) }) @@ -3018,16 +3019,18 @@ describe.each([ ) }) - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $and: { - conditions: [{ equal: { name: "" } }], + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + $and: { + conditions: [{ equal: { name: "" } }], + }, }, - }, - }).toFindNothing() - }) + }).toFindNothing() + }) it("returns all rows when onEmptyFilter set to all", async () => { await expectSearch({ @@ -3168,16 +3171,18 @@ describe.each([ }).toContainExactly([{ age: 1, name: "Jane" }]) }) - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $or: { - conditions: [{ equal: { name: "" } }], + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + $or: { + conditions: [{ equal: { name: "" } }], + }, }, - }, - }).toFindNothing() - }) + }).toFindNothing() + }) it("returns all rows when onEmptyFilter set to all", async () => { await expectSearch({ diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index fd003ff56a..e83ea381b5 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -113,11 +113,13 @@ export async function search( options.query = query } else { options.query = { - onEmptyFilter: viewQuery.onEmptyFilter, $and: { conditions: [viewQuery, options.query], }, } + if (viewQuery.onEmptyFilter) { + options.query.onEmptyFilter = viewQuery.onEmptyFilter + } } } From 39511a4e83dc3481697e638e4c73848589dd1621 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 2 Oct 2024 17:39:35 +0200 Subject: [PATCH 094/105] Fix tests --- .../src/api/routes/tests/search.spec.ts | 111 ++++++++---------- 1 file changed, 48 insertions(+), 63 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 679ff0e356..037d8b3fe8 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -70,29 +70,23 @@ describe.each([ let rows: Row[] async function basicRelationshipTables(type: RelationshipType) { - const relatedTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - generator.guid().substring(0, 10) - ) - const tableId = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - //@ts-ignore - API accepts this structure, will build out rest of definition - productCat: { - type: FieldType.LINK, - relationshipType: type, - name: "productCat", - fieldName: "product", - tableId: relatedTable, - constraints: { - type: "array", - }, + const relatedTable = await createTable({ + name: { name: "name", type: FieldType.STRING }, + }) + const tableId = await createTable({ + name: { name: "name", type: FieldType.STRING }, + //@ts-ignore - API accepts this structure, will build out rest of definition + productCat: { + type: FieldType.LINK, + relationshipType: type, + name: "productCat", + fieldName: "product", + tableId: relatedTable, + constraints: { + type: "array", }, }, - generator.guid().substring(0, 10) - ) + }) return { relatedTable: await config.api.table.get(relatedTable), tableId, @@ -138,9 +132,9 @@ describe.each([ } }) - async function createTable(schema: TableSchema, name?: string) { + async function createTable(schema: TableSchema) { const table = await config.api.table.save( - tableForDatasource(datasource, { schema, name }) + tableForDatasource(datasource, { schema }) ) return table._id! } @@ -157,8 +151,8 @@ describe.each([ ["table", createTable], [ "view", - async (schema: TableSchema, name?: string) => { - const tableId = await createTable(schema, name) + async (schema: TableSchema) => { + const tableId = await createTable(schema) const view = await config.api.viewV2.create({ tableId: tableId, name: generator.guid(), @@ -2042,42 +2036,36 @@ describe.each([ isSql && describe("related formulas", () => { beforeAll(async () => { - const arrayTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - array: { - name: "array", - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - inclusion: ["option 1", "option 2"], - }, + const arrayTable = await createTable({ + name: { name: "name", type: FieldType.STRING }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["option 1", "option 2"], }, }, - "array" - ) - tableOrViewId = await createTableOrView( - { - relationship: { - type: FieldType.LINK, - relationshipType: RelationshipType.MANY_TO_ONE, - name: "relationship", - fieldName: "relate", - tableId: arrayTable, - constraints: { - type: "array", - }, - }, - formula: { - type: FieldType.FORMULA, - name: "formula", - formula: encodeJSBinding( - `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` - ), + }) + tableOrViewId = await createTableOrView({ + relationship: { + type: FieldType.LINK, + relationshipType: RelationshipType.MANY_TO_ONE, + name: "relationship", + fieldName: "relate", + tableId: arrayTable, + constraints: { + type: "array", }, }, - "main" - ) + formula: { + type: FieldType.FORMULA, + name: "formula", + formula: encodeJSBinding( + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` + ), + }, + }) const arrayRows = await Promise.all([ config.api.row.save(arrayTable, { name: "foo", @@ -2378,12 +2366,9 @@ describe.each([ let relatedTable: string, relatedRows: Row[] beforeAll(async () => { - relatedTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) + relatedTable = await createTable({ + name: { name: "name", type: FieldType.STRING }, + }) tableOrViewId = await createTableOrView({ name: { name: "name", type: FieldType.STRING }, related1: { From 73613ce8bfda5720aca62a3a65dd35ca97d3113e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 2 Oct 2024 16:52:40 +0100 Subject: [PATCH 095/105] Fix spread operator use. --- packages/server/src/sdk/app/views/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 2bd90822c7..83ee78e165 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -307,7 +307,7 @@ export async function enrichSchema( // if nothing specified in view, then it is not visible const ui = viewSchema[key] || { visible: false } schema[key] = { - ...(tableSchema[key] || {}), + ...tableSchema[key], ...ui, order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order, columns: undefined, From 9e7ed0471905eaa5eb5ba3fdf7444c733e6c7858 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:43:07 +0100 Subject: [PATCH 096/105] Support saving queryUI. --- packages/server/src/api/controllers/view/viewsV2.ts | 6 ++++-- packages/types/src/api/web/app/view.ts | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 3df7172de2..0257c86ded 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,10 +99,11 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version" | "queryUI"> = { + const parsedView: Omit, "id" | "version"> = { name: view.name, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, @@ -132,12 +133,13 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys> = { + const parsedView: RequiredKeys = { id: view.id, name: view.name, version: view.version, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a99f2938ab..a6be5e2986 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,7 +9,6 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest - extends Omit {} +export interface CreateViewRequest extends Omit {} -export interface UpdateViewRequest extends Omit {} +export interface UpdateViewRequest extends ViewV2 {} From 98ebd8f18cf45a026aad06a5b31bc0b538440005 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:59:20 +0100 Subject: [PATCH 097/105] Omitting in tests. --- packages/server/src/api/routes/tests/viewV2.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..da2354017a 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -154,7 +154,7 @@ describe.each([ }) it("can persist views with all fields", async () => { - const newView: Required = { + const newView: Required> = { name: generator.name(), tableId: table._id!, primaryDisplay: "id", @@ -584,7 +584,7 @@ describe.each([ it("can update all fields", async () => { const tableId = table._id! - const updatedData: Required = { + const updatedData: Required> = { version: view.version, id: view.id, tableId, From 7c778061e5883685172884240673a72db3615b1b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 3 Oct 2024 09:50:25 +0200 Subject: [PATCH 098/105] Fix sql relationship fetching --- packages/server/src/api/controllers/row/utils/sqlUtils.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 36521b10c6..607fee7580 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -124,9 +124,11 @@ export async function buildSqlFieldList( ([columnName, column]) => column.type !== FieldType.LINK && column.type !== FieldType.FORMULA && - !existing.find((field: string) => field === columnName) + !existing.find( + (field: string) => field === `${table.name}.${columnName}` + ) ) - .map(column => `${table.name}.${column[0]}`) + .map(([columnName]) => `${table.name}.${columnName}`) } let fields: string[] = [] From 1b53bab92ea88730d553c8e4ce3704beee984c98 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 3 Oct 2024 10:28:29 +0200 Subject: [PATCH 099/105] Extract --- .../src/api/routes/tests/search.spec.ts | 37 +++++++++++-------- .../src/api/routes/tests/viewV2.spec.ts | 7 +--- packages/server/src/db/linkedRows/index.ts | 4 +- packages/types/src/documents/app/view.ts | 4 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 037d8b3fe8..d6c149bf3f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -34,7 +34,7 @@ import { Table, TableSchema, User, - ViewFieldMetadata, + ViewV2Schema, } from "@budibase/types" import _ from "lodash" import tk from "timekeeper" @@ -139,6 +139,15 @@ describe.each([ return table._id! } + async function createView(tableId: string, schema: ViewV2Schema) { + const view = await config.api.viewV2.create({ + tableId: tableId, + name: generator.guid(), + schema, + }) + return view + } + async function createRows(arr: Record[]) { // Shuffling to avoid false positives given a fixed order for (const row of _.shuffle(arr)) { @@ -153,21 +162,17 @@ describe.each([ "view", async (schema: TableSchema) => { const tableId = await createTable(schema) - const view = await config.api.viewV2.create({ - tableId: tableId, - name: generator.guid(), - schema: Object.keys(schema).reduce>( - (viewSchema, fieldName) => { - const field = schema[fieldName] - viewSchema[fieldName] = { - visible: field.visible ?? true, - readonly: false, - } - return viewSchema - }, - {} - ), - }) + const view = await createView( + tableId, + Object.keys(schema).reduce((viewSchema, fieldName) => { + const field = schema[fieldName] + viewSchema[fieldName] = { + visible: field.visible ?? true, + readonly: false, + } + return viewSchema + }, {}) + ) return view.id }, ], diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..f63d8aeb85 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,9 +22,9 @@ import { RelationshipType, TableSchema, RenameColumn, - ViewFieldMetadata, FeatureFlag, BBReferenceFieldSubType, + ViewV2Schema, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -1152,10 +1152,7 @@ describe.each([ return table } - const createView = async ( - tableId: string, - schema: Record - ) => + const createView = async (tableId: string, schema: ViewV2Schema) => await config.api.viewV2.create({ name: generator.guid(), tableId, diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 45f5ee6e5a..f359bcc2cf 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -23,8 +23,8 @@ import { Row, Table, TableSchema, - ViewFieldMetadata, ViewV2, + ViewV2Schema, } from "@budibase/types" import sdk from "../../sdk" import { helpers } from "@budibase/shared-core" @@ -262,7 +262,7 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} + let viewSchema: ViewV2Schema = {} if (sdk.views.isView(source)) { if (helpers.views.isCalculationView(source)) { return enriched diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index a957564039..271df45c65 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -71,9 +71,11 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: ViewV2Schema } +export type ViewV2Schema = Record + export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema export interface ViewCountOrSumSchema { From 36e304aaf5d7bb1df2ff835f0fc84f7ea4680182 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 3 Oct 2024 10:44:15 +0200 Subject: [PATCH 100/105] Unify --- packages/server/src/api/routes/tests/search.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index d6c149bf3f..cf3e2f6c1b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -145,7 +145,7 @@ describe.each([ name: generator.guid(), schema, }) - return view + return view.id } async function createRows(arr: Record[]) { @@ -162,7 +162,7 @@ describe.each([ "view", async (schema: TableSchema) => { const tableId = await createTable(schema) - const view = await createView( + const viewId = await createView( tableId, Object.keys(schema).reduce((viewSchema, fieldName) => { const field = schema[fieldName] @@ -173,7 +173,7 @@ describe.each([ return viewSchema }, {}) ) - return view.id + return viewId }, ], ])("from %s", (tableOrView, createTableOrView) => { From 9ba759093ebd5aeefdb45e705ed8cf295aa04b0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 3 Oct 2024 10:44:58 +0200 Subject: [PATCH 101/105] Small rename --- packages/server/src/api/routes/tests/search.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index cf3e2f6c1b..110899e292 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -176,8 +176,8 @@ describe.each([ return viewId }, ], - ])("from %s", (tableOrView, createTableOrView) => { - const isView = tableOrView === "view" + ])("from %s", (sourceType, createTableOrView) => { + const isView = sourceType === "view" if (isView && isLucene) { // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests From 9440dc98edec736ea902b3531719b770e9889dd7 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 3 Oct 2024 09:15:38 +0000 Subject: [PATCH 102/105] Bump version to 2.32.11 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 092e9a133e..a4bcb56d38 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.10", + "version": "2.32.11", "npmClient": "yarn", "packages": [ "packages/*", From dfb991b93e0bf9770d009a7fef89349b6d0dd499 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 3 Oct 2024 11:31:14 +0200 Subject: [PATCH 103/105] Fix oss yarn.lock --- yarn.lock | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/yarn.lock b/yarn.lock index b95f6f4f0f..1198e98ad6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2051,7 +2051,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.32.10": +"@budibase/backend-core@2.32.11": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2132,15 +2132,15 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "2.32.10" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5" - integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ== + version "2.32.11" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d" + integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA== dependencies: "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "2.32.10" - "@budibase/shared-core" "2.32.10" - "@budibase/string-templates" "2.32.10" - "@budibase/types" "2.32.10" + "@budibase/backend-core" "2.32.11" + "@budibase/shared-core" "2.32.11" + "@budibase/string-templates" "2.32.11" + "@budibase/types" "2.32.11" "@koa/router" "8.0.8" bull "4.10.1" dd-trace "5.2.0" @@ -2153,13 +2153,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.32.10": +"@budibase/shared-core@2.32.11": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@2.32.10": +"@budibase/string-templates@2.32.11": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2167,7 +2167,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@2.32.10": +"@budibase/types@2.32.11": version "0.0.0" dependencies: scim-patch "^0.8.1" From 9c70ed92ba58094abba9b53d28a991f3e89db559 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 14:47:31 +0100 Subject: [PATCH 104/105] Small build fix. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 7e73a51889..1a15eb5839 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -92,7 +92,7 @@ export async function search( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQueryLegacy(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query) || {} delete viewQuery?.onEmptyFilter const sqsEnabled = await features.flags.isEnabled("SQS") From 5431becf01e0d53b75e366819b9f70dd083829c4 Mon Sep 17 00:00:00 2001 From: Christos Alexiou Date: Thu, 3 Oct 2024 18:44:41 +0300 Subject: [PATCH 105/105] DEPLOY expects boolean on the other end --- .github/workflows/deploy-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index a676fe75f0..463074e836 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -23,7 +23,7 @@ jobs: PAYLOAD_BRANCH: ${{ github.head_ref }} PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} PAYLOAD_LICENSE_TYPE: "free" - PAYLOAD_DEPLOY: "true" + PAYLOAD_DEPLOY: true with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy