diff --git a/hosting/nginx.dev.conf b/hosting/nginx.dev.conf index e3d6d47287..f0a58a9a98 100644 --- a/hosting/nginx.dev.conf +++ b/hosting/nginx.dev.conf @@ -45,20 +45,6 @@ http { client_max_body_size 50000m; ignore_invalid_headers off; proxy_buffering off; - set $csp_default "default-src 'self'"; - set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com"; - set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; - set $csp_object "object-src 'none'"; - set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; - set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; - set $csp_frame "frame-src 'self' https:"; - set $csp_img "img-src http: https: data: blob:"; - set $csp_manifest "manifest-src 'self'"; - set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live"; - set $csp_worker "worker-src blob:"; - - add_header Content-Security-Policy "${csp_default}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always; error_page 502 503 504 /error.html; location = /error.html { diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index b807db0ee3..83b9b69d0b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -10,7 +10,6 @@ import { DatabaseQueryOpts, DBError, Document, - FeatureFlag, isDocument, RowResponse, RowValue, @@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import { sqlLog } from "../../sql/utils" -import { flags } from "../../features" const DATABASE_NOT_FOUND = "Database does not exist." @@ -456,10 +454,7 @@ export class DatabaseImpl implements Database { } async destroy() { - if ( - (await flags.isEnabled(FeatureFlag.SQS)) && - (await this.exists(SQLITE_DESIGN_DOC_ID)) - ) { + if (await this.exists(SQLITE_DESIGN_DOC_ID)) { // delete the design document, then run the cleanup operation const definition = await this.get(SQLITE_DESIGN_DOC_ID) // remove all tables - save the definition then trigger a cleanup diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index de6b9cad8b..b3f016e88a 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,8 +269,6 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true), [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true), - [FeatureFlag.SQS]: Flag.boolean(true), - [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true), [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true), }) diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte index 85af8596a0..2cda43dda0 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte @@ -11,7 +11,6 @@ export let disabledPermissions = [] export let columns export let fromRelationshipField - export let canSetRelationshipSchemas const { datasource, dispatch } = getContext("grid") @@ -129,6 +128,8 @@ } }) + $: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK) + async function toggleColumn(column, permission) { const visible = permission !== FieldPermissions.HIDDEN const readonly = permission === FieldPermissions.READONLY @@ -184,7 +185,7 @@ value={columnToPermissionOptions(column)} options={column.options} /> - {#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN} + {#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
{ @@ -203,7 +204,7 @@
-{#if canSetRelationshipSchemas} +{#if hasLinkColumns} (relationshipFieldName = null)} open={relationshipFieldName} diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte index 7ab84ba039..801abeaf9f 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte @@ -10,8 +10,6 @@ import { getContext } from "svelte" import { ActionButton } from "@budibase/bbui" import ColumnsSettingContent from "./ColumnsSettingContent.svelte" - import { isEnabled } from "helpers/featureFlags" - import { FeatureFlag } from "@budibase/types" import DetailPopover from "components/common/DetailPopover.svelte" const { tableColumns, datasource } = getContext("grid") @@ -46,9 +44,5 @@ {text} - + diff --git a/packages/pro b/packages/pro index 80770215c6..a56696a4af 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 80770215c6159e4d47f3529fd02e74bc8ad07543 +Subproject commit a56696a4af5667617746600fc75fe6a01744b692 diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 04e77fbe62..d6b29dc12c 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils" import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { context, events, features, HTTPError } from "@budibase/backend-core" +import { context, events, HTTPError } from "@budibase/backend-core" import { AutoFieldSubType, Database, Datasource, - FeatureFlag, FieldSchema, FieldType, NumberFieldMetadata, @@ -336,9 +335,8 @@ class TableSaveFunctions { importRows: this.importRows, userId: this.userId, }) - if (await features.flags.isEnabled(FeatureFlag.SQS)) { - await sdk.tables.sqs.addTable(table) - } + + await sdk.tables.sqs.addTable(table) return table } @@ -530,9 +528,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) { if (rows) { await AttachmentCleanup.tableDelete(table, rows) } - if (await features.flags.isEnabled(FeatureFlag.SQS)) { - await sdk.tables.sqs.removeTable(table) - } + + await sdk.tables.sqs.removeTable(table) } const _TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index 6d85cdbda9..1511c1aa61 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -16,7 +16,7 @@ jest.mock("../../../utilities/redis", () => ({ import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" -import { events, utils, context, features } from "@budibase/backend-core" +import { events, utils, context } from "@budibase/backend-core" import env from "../../../environment" import { type App, BuiltinPermissionID } from "@budibase/types" import tk from "timekeeper" @@ -355,21 +355,6 @@ describe("/applications", () => { expect(events.app.deleted).toHaveBeenCalledTimes(1) expect(events.app.unpublished).toHaveBeenCalledTimes(1) }) - - it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => { - const prodAppId = app.appId.replace("_dev", "") - nock("http://localhost:10000") - .delete(`/api/global/roles/${prodAppId}`) - .reply(200, {}) - - await features.testutils.withFeatureFlags( - "*", - { SQS: true }, - async () => { - await config.api.application.delete(app.appId) - } - ) - }) }) describe("POST /api/applications/:appId/duplicate", () => { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index bf8f5a2a1c..6cb7def670 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -9,27 +9,18 @@ import { import tk from "timekeeper" import emitter from "../../../../src/events" import { outputProcessing } from "../../../utilities/rowProcessor" -import { - context, - InternalTable, - tenancy, - features, - utils, -} from "@budibase/backend-core" +import { context, InternalTable, tenancy, utils } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { AIOperationEnum, - AttachmentFieldMetadata, AutoFieldSubType, Datasource, - DateFieldMetadata, DeleteRow, FieldSchema, FieldType, BBReferenceFieldSubType, FormulaType, INTERNAL_TABLE_SOURCE_ID, - NumberFieldMetadata, QuotaUsageType, RelationshipType, Row, @@ -90,8 +81,7 @@ async function waitForEvent( } describe.each([ - ["lucene", undefined], - ["sqs", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], @@ -99,8 +89,6 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined - const isLucene = providerType === "lucene" - const isSqs = providerType === "sqs" const isMSSQL = providerType === DatabaseName.SQL_SERVER const isOracle = providerType === DatabaseName.ORACLE const config = setup.getConfig() @@ -108,15 +96,9 @@ describe.each([ let table: Table let datasource: Datasource | undefined let client: Knex | undefined - let envCleanup: (() => void) | undefined beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: true }, () => - config.init() - ) - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) + await config.init() if (dsProvider) { const rawDatasource = await dsProvider @@ -129,9 +111,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) function saveTableRequest( @@ -381,185 +360,6 @@ describe.each([ expect(ids).toEqual(expect.arrayContaining(sequence)) }) - isLucene && - it("row values are coerced", async () => { - const str: FieldSchema = { - type: FieldType.STRING, - name: "str", - constraints: { type: "string", presence: false }, - } - const singleAttachment: FieldSchema = { - type: FieldType.ATTACHMENT_SINGLE, - name: "single attachment", - constraints: { presence: false }, - } - const attachmentList: AttachmentFieldMetadata = { - type: FieldType.ATTACHMENTS, - name: "attachments", - constraints: { type: "array", presence: false }, - } - const signature: FieldSchema = { - type: FieldType.SIGNATURE_SINGLE, - name: "signature", - constraints: { presence: false }, - } - const bool: FieldSchema = { - type: FieldType.BOOLEAN, - name: "boolean", - constraints: { type: "boolean", presence: false }, - } - const number: NumberFieldMetadata = { - type: FieldType.NUMBER, - name: "str", - constraints: { type: "number", presence: false }, - } - const datetime: DateFieldMetadata = { - type: FieldType.DATETIME, - name: "datetime", - constraints: { - type: "string", - presence: false, - datetime: { earliest: "", latest: "" }, - }, - } - const arrayField: FieldSchema = { - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - presence: false, - inclusion: ["One", "Two", "Three"], - }, - name: "Sample Tags", - sortable: false, - } - const optsField: FieldSchema = { - name: "Sample Opts", - type: FieldType.OPTIONS, - constraints: { - type: "string", - presence: false, - inclusion: ["Alpha", "Beta", "Gamma"], - }, - } - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: str, - stringUndefined: str, - stringNull: str, - stringString: str, - numberEmptyString: number, - numberNull: number, - numberUndefined: number, - numberString: number, - numberNumber: number, - datetimeEmptyString: datetime, - datetimeNull: datetime, - datetimeUndefined: datetime, - datetimeString: datetime, - datetimeDate: datetime, - boolNull: bool, - boolEmpty: bool, - boolUndefined: bool, - boolString: bool, - boolBool: bool, - singleAttachmentNull: singleAttachment, - singleAttachmentUndefined: singleAttachment, - attachmentListNull: attachmentList, - attachmentListUndefined: attachmentList, - attachmentListEmpty: attachmentList, - attachmentListEmptyArrayStr: attachmentList, - signatureNull: signature, - signatureUndefined: signature, - arrayFieldEmptyArrayStr: arrayField, - arrayFieldArrayStrKnown: arrayField, - arrayFieldNull: arrayField, - arrayFieldUndefined: arrayField, - optsFieldEmptyStr: optsField, - optsFieldUndefined: optsField, - optsFieldNull: optsField, - optsFieldStrKnown: optsField, - }, - }) - ) - - const datetimeStr = "1984-04-20T00:00:00.000Z" - - const row = await config.api.row.save(table._id!, { - name: "Test Row", - stringUndefined: undefined, - stringNull: null, - stringString: "i am a string", - numberEmptyString: "", - numberNull: null, - numberUndefined: undefined, - numberString: "123", - numberNumber: 123, - datetimeEmptyString: "", - datetimeNull: null, - datetimeUndefined: undefined, - datetimeString: datetimeStr, - datetimeDate: new Date(datetimeStr), - boolNull: null, - boolEmpty: "", - boolUndefined: undefined, - boolString: "true", - boolBool: true, - tableId: table._id, - singleAttachmentNull: null, - singleAttachmentUndefined: undefined, - attachmentListNull: null, - attachmentListUndefined: undefined, - attachmentListEmpty: "", - attachmentListEmptyArrayStr: "[]", - signatureNull: null, - signatureUndefined: undefined, - arrayFieldEmptyArrayStr: "[]", - arrayFieldUndefined: undefined, - arrayFieldNull: null, - arrayFieldArrayStrKnown: "['One']", - optsFieldEmptyStr: "", - optsFieldUndefined: undefined, - optsFieldNull: null, - optsFieldStrKnown: "Alpha", - }) - - expect(row.stringUndefined).toBe(undefined) - expect(row.stringNull).toBe(null) - expect(row.stringString).toBe("i am a string") - expect(row.numberEmptyString).toBe(null) - expect(row.numberNull).toBe(null) - expect(row.numberUndefined).toBe(undefined) - expect(row.numberString).toBe(123) - expect(row.numberNumber).toBe(123) - expect(row.datetimeEmptyString).toBe(null) - expect(row.datetimeNull).toBe(null) - expect(row.datetimeUndefined).toBe(undefined) - expect(row.datetimeString).toBe(new Date(datetimeStr).toISOString()) - expect(row.datetimeDate).toBe(new Date(datetimeStr).toISOString()) - expect(row.boolNull).toBe(null) - expect(row.boolEmpty).toBe(null) - expect(row.boolUndefined).toBe(undefined) - expect(row.boolString).toBe(true) - expect(row.boolBool).toBe(true) - expect(row.singleAttachmentNull).toEqual(null) - expect(row.singleAttachmentUndefined).toBe(undefined) - expect(row.attachmentListNull).toEqual([]) - expect(row.attachmentListUndefined).toBe(undefined) - expect(row.attachmentListEmpty).toEqual([]) - expect(row.attachmentListEmptyArrayStr).toEqual([]) - expect(row.signatureNull).toEqual(null) - expect(row.signatureUndefined).toBe(undefined) - expect(row.arrayFieldEmptyArrayStr).toEqual([]) - expect(row.arrayFieldNull).toEqual([]) - expect(row.arrayFieldUndefined).toEqual(undefined) - expect(row.optsFieldEmptyStr).toEqual(null) - expect(row.optsFieldUndefined).toEqual(undefined) - expect(row.optsFieldNull).toEqual(null) - expect(row.arrayFieldArrayStrKnown).toEqual(["One"]) - expect(row.optsFieldStrKnown).toEqual("Alpha") - }) - isInternal && it("doesn't allow creating in user table", async () => { const response = await config.api.row.save( @@ -1023,104 +823,103 @@ describe.each([ }) }) - !isLucene && - describe("relations to same table", () => { - let relatedRows: Row[] + describe("relations to same table", () => { + let relatedRows: Row[] - beforeAll(async () => { - const relatedTable = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, + beforeAll(async () => { + const relatedTable = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + const relatedTableId = relatedTable._id! + table = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, }, - }) - ) - const relatedTableId = relatedTable._id! - table = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, - related1: { - type: FieldType.LINK, - name: "related1", - fieldName: "main1", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - related2: { - type: FieldType.LINK, - name: "related2", - fieldName: "main2", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, + related2: { + type: FieldType.LINK, + name: "related2", + fieldName: "main2", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, }, - }) - ) - relatedRows = await Promise.all([ - config.api.row.save(relatedTableId, { name: "foo" }), - config.api.row.save(relatedTableId, { name: "bar" }), - config.api.row.save(relatedTableId, { name: "baz" }), - config.api.row.save(relatedTableId, { name: "boo" }), - ]) - }) - - it("can create rows with both relationships", async () => { - const row = await config.api.row.save(table._id!, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], + }, }) - - expect(row).toEqual( - expect.objectContaining({ - name: "test", - related1: [ - { - _id: relatedRows[0]._id, - primaryDisplay: relatedRows[0].name, - }, - ], - related2: [ - { - _id: relatedRows[1]._id, - primaryDisplay: relatedRows[1].name, - }, - ], - }) - ) - }) - - it("can create rows with no relationships", async () => { - const row = await config.api.row.save(table._id!, { - name: "test", - }) - - expect(row.related1).toBeUndefined() - expect(row.related2).toBeUndefined() - }) - - it("can create rows with only one relationships field", async () => { - const row = await config.api.row.save(table._id!, { - name: "test", - related1: [], - related2: [relatedRows[1]._id!], - }) - - expect(row).toEqual( - expect.objectContaining({ - name: "test", - related2: [ - { - _id: relatedRows[1]._id, - primaryDisplay: relatedRows[1].name, - }, - ], - }) - ) - expect(row.related1).toBeUndefined() - }) + ) + relatedRows = await Promise.all([ + config.api.row.save(relatedTableId, { name: "foo" }), + config.api.row.save(relatedTableId, { name: "bar" }), + config.api.row.save(relatedTableId, { name: "baz" }), + config.api.row.save(relatedTableId, { name: "boo" }), + ]) }) + + it("can create rows with both relationships", async () => { + const row = await config.api.row.save(table._id!, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }) + + expect(row).toEqual( + expect.objectContaining({ + name: "test", + related1: [ + { + _id: relatedRows[0]._id, + primaryDisplay: relatedRows[0].name, + }, + ], + related2: [ + { + _id: relatedRows[1]._id, + primaryDisplay: relatedRows[1].name, + }, + ], + }) + ) + }) + + it("can create rows with no relationships", async () => { + const row = await config.api.row.save(table._id!, { + name: "test", + }) + + expect(row.related1).toBeUndefined() + expect(row.related2).toBeUndefined() + }) + + it("can create rows with only one relationships field", async () => { + const row = await config.api.row.save(table._id!, { + name: "test", + related1: [], + related2: [relatedRows[1]._id!], + }) + + expect(row).toEqual( + expect.objectContaining({ + name: "test", + related2: [ + { + _id: relatedRows[1]._id, + primaryDisplay: relatedRows[1].name, + }, + ], + }) + ) + expect(row.related1).toBeUndefined() + }) + }) }) describe("get", () => { @@ -1224,133 +1023,132 @@ describe.each([ expect(rows).toHaveLength(1) }) - !isLucene && - describe("relations to same table", () => { - let relatedRows: Row[] + describe("relations to same table", () => { + let relatedRows: Row[] - beforeAll(async () => { - const relatedTable = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, + beforeAll(async () => { + const relatedTable = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + const relatedTableId = relatedTable._id! + table = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, }, - }) - ) - const relatedTableId = relatedTable._id! - table = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, - related1: { - type: FieldType.LINK, - name: "related1", - fieldName: "main1", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - related2: { - type: FieldType.LINK, - name: "related2", - fieldName: "main2", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, + related2: { + type: FieldType.LINK, + name: "related2", + fieldName: "main2", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, }, - }) - ) - relatedRows = await Promise.all([ - config.api.row.save(relatedTableId, { name: "foo" }), - config.api.row.save(relatedTableId, { name: "bar" }), - config.api.row.save(relatedTableId, { name: "baz" }), - config.api.row.save(relatedTableId, { name: "boo" }), - ]) - }) - - it("can edit rows with both relationships", async () => { - let row = await config.api.row.save(table._id!, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], + }, }) - - row = await config.api.row.save(table._id!, { - ...row, - related1: [relatedRows[0]._id!, relatedRows[1]._id!], - related2: [relatedRows[2]._id!], - }) - - expect(row).toEqual( - expect.objectContaining({ - name: "test", - related1: expect.arrayContaining([ - { - _id: relatedRows[0]._id, - primaryDisplay: relatedRows[0].name, - }, - { - _id: relatedRows[1]._id, - primaryDisplay: relatedRows[1].name, - }, - ]), - related2: [ - { - _id: relatedRows[2]._id, - primaryDisplay: relatedRows[2].name, - }, - ], - }) - ) - }) - - it("can drop existing relationship", async () => { - let row = await config.api.row.save(table._id!, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], - }) - - row = await config.api.row.save(table._id!, { - ...row, - related1: [], - related2: [relatedRows[2]._id!], - }) - - expect(row).toEqual( - expect.objectContaining({ - name: "test", - related2: [ - { - _id: relatedRows[2]._id, - primaryDisplay: relatedRows[2].name, - }, - ], - }) - ) - expect(row.related1).toBeUndefined() - }) - - it("can drop both relationships", async () => { - let row = await config.api.row.save(table._id!, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], - }) - - row = await config.api.row.save(table._id!, { - ...row, - related1: [], - related2: [], - }) - - expect(row).toEqual( - expect.objectContaining({ - name: "test", - }) - ) - expect(row.related1).toBeUndefined() - expect(row.related2).toBeUndefined() - }) + ) + relatedRows = await Promise.all([ + config.api.row.save(relatedTableId, { name: "foo" }), + config.api.row.save(relatedTableId, { name: "bar" }), + config.api.row.save(relatedTableId, { name: "baz" }), + config.api.row.save(relatedTableId, { name: "boo" }), + ]) }) + + it("can edit rows with both relationships", async () => { + let row = await config.api.row.save(table._id!, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }) + + row = await config.api.row.save(table._id!, { + ...row, + related1: [relatedRows[0]._id!, relatedRows[1]._id!], + related2: [relatedRows[2]._id!], + }) + + expect(row).toEqual( + expect.objectContaining({ + name: "test", + related1: expect.arrayContaining([ + { + _id: relatedRows[0]._id, + primaryDisplay: relatedRows[0].name, + }, + { + _id: relatedRows[1]._id, + primaryDisplay: relatedRows[1].name, + }, + ]), + related2: [ + { + _id: relatedRows[2]._id, + primaryDisplay: relatedRows[2].name, + }, + ], + }) + ) + }) + + it("can drop existing relationship", async () => { + let row = await config.api.row.save(table._id!, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }) + + row = await config.api.row.save(table._id!, { + ...row, + related1: [], + related2: [relatedRows[2]._id!], + }) + + expect(row).toEqual( + expect.objectContaining({ + name: "test", + related2: [ + { + _id: relatedRows[2]._id, + primaryDisplay: relatedRows[2].name, + }, + ], + }) + ) + expect(row.related1).toBeUndefined() + }) + + it("can drop both relationships", async () => { + let row = await config.api.row.save(table._id!, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }) + + row = await config.api.row.save(table._id!, { + ...row, + related1: [], + related2: [], + }) + + expect(row).toEqual( + expect.objectContaining({ + name: "test", + }) + ) + expect(row.related1).toBeUndefined() + expect(row.related2).toBeUndefined() + }) + }) }) describe("patch", () => { @@ -1628,72 +1426,71 @@ describe.each([ expect(res.length).toEqual(2) }) - !isLucene && - describe("relations to same table", () => { - let relatedRows: Row[] + describe("relations to same table", () => { + let relatedRows: Row[] - beforeAll(async () => { - const relatedTable = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, - }, - }) - ) - const relatedTableId = relatedTable._id! - table = await config.api.table.save( - defaultTable({ - schema: { - name: { name: "name", type: FieldType.STRING }, - related1: { - type: FieldType.LINK, - name: "related1", - fieldName: "main1", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - related2: { - type: FieldType.LINK, - name: "related2", - fieldName: "main2", - tableId: relatedTableId, - relationshipType: RelationshipType.MANY_TO_MANY, - }, - }, - }) - ) - relatedRows = await Promise.all([ - config.api.row.save(relatedTableId, { name: "foo" }), - config.api.row.save(relatedTableId, { name: "bar" }), - config.api.row.save(relatedTableId, { name: "baz" }), - config.api.row.save(relatedTableId, { name: "boo" }), - ]) - }) - - it("can delete rows with both relationships", async () => { - const row = await config.api.row.save(table._id!, { - name: "test", - related1: [relatedRows[0]._id!], - related2: [relatedRows[1]._id!], + beforeAll(async () => { + const relatedTable = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + }, }) - - await config.api.row.delete(table._id!, { _id: row._id! }) - - await config.api.row.get(table._id!, row._id!, { status: 404 }) - }) - - it("can delete rows with empty relationships", async () => { - const row = await config.api.row.save(table._id!, { - name: "test", - related1: [], - related2: [], + ) + const relatedTableId = relatedTable._id! + table = await config.api.table.save( + defaultTable({ + schema: { + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + related2: { + type: FieldType.LINK, + name: "related2", + fieldName: "main2", + tableId: relatedTableId, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }, }) - - await config.api.row.delete(table._id!, { _id: row._id! }) - - await config.api.row.get(table._id!, row._id!, { status: 404 }) - }) + ) + relatedRows = await Promise.all([ + config.api.row.save(relatedTableId, { name: "foo" }), + config.api.row.save(relatedTableId, { name: "bar" }), + config.api.row.save(relatedTableId, { name: "baz" }), + config.api.row.save(relatedTableId, { name: "boo" }), + ]) }) + + it("can delete rows with both relationships", async () => { + const row = await config.api.row.save(table._id!, { + name: "test", + related1: [relatedRows[0]._id!], + related2: [relatedRows[1]._id!], + }) + + await config.api.row.delete(table._id!, { _id: row._id! }) + + await config.api.row.get(table._id!, row._id!, { status: 404 }) + }) + + it("can delete rows with empty relationships", async () => { + const row = await config.api.row.save(table._id!, { + name: "test", + related1: [], + related2: [], + }) + + await config.api.row.delete(table._id!, { _id: row._id! }) + + await config.api.row.get(table._id!, row._id!, { status: 404 }) + }) + }) }) describe("validate", () => { @@ -3061,13 +2858,7 @@ describe.each([ let auxData: Row[] = [] - let flagCleanup: (() => void) | undefined - beforeAll(async () => { - flagCleanup = features.testutils.setFeatureFlags("*", { - ENRICHED_RELATIONSHIPS: true, - }) - const aux2Table = await config.api.table.save(saveTableRequest()) const aux2Data = await config.api.row.save(aux2Table._id!, {}) @@ -3214,10 +3005,6 @@ describe.each([ viewId = view.id }) - afterAll(() => { - flagCleanup?.() - }) - const testScenarios: [string, (row: Row) => Promise | Row][] = [ ["get row", (row: Row) => config.api.row.get(viewId, row._id!)], [ @@ -3290,68 +3077,6 @@ describe.each([ } ) - it.each(testScenarios)( - "does not enrich relationships when not enabled (via %s)", - async (__, retrieveDelegate) => { - await features.testutils.withFeatureFlags( - "*", - { - ENRICHED_RELATIONSHIPS: false, - }, - async () => { - const otherRows = _.sampleSize(auxData, 5) - - const row = await config.api.row.save(viewId, { - title: generator.word(), - relWithNoSchema: [otherRows[0]], - relWithEmptySchema: [otherRows[1]], - relWithFullSchema: [otherRows[2]], - relWithHalfSchema: [otherRows[3]], - relWithIllegalSchema: [otherRows[4]], - }) - - const retrieved = await retrieveDelegate(row) - - expect(retrieved).toEqual( - expect.objectContaining({ - title: row.title, - relWithNoSchema: [ - { - _id: otherRows[0]._id, - primaryDisplay: otherRows[0].name, - }, - ], - relWithEmptySchema: [ - { - _id: otherRows[1]._id, - primaryDisplay: otherRows[1].name, - }, - ], - relWithFullSchema: [ - { - _id: otherRows[2]._id, - primaryDisplay: otherRows[2].name, - }, - ], - relWithHalfSchema: [ - { - _id: otherRows[3]._id, - primaryDisplay: otherRows[3].name, - }, - ], - relWithIllegalSchema: [ - { - _id: otherRows[4]._id, - primaryDisplay: otherRows[4].name, - }, - ], - }) - ) - } - ) - } - ) - it.each([ [ "from table fetch", @@ -3422,7 +3147,7 @@ describe.each([ ) }) - isSqs && + isInternal && describe("AI fields", () => { let table: Table diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 2da213cd38..8d06c3c4ef 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -8,7 +8,6 @@ import { context, db as dbCore, docIds, - features, MAX_VALID_DATE, MIN_VALID_DATE, SQLITE_DESIGN_DOC_ID, @@ -64,7 +63,6 @@ jest.mock("@budibase/pro", () => ({ describe.each([ ["in-memory", undefined], - ["lucene", undefined], ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], @@ -72,15 +70,12 @@ describe.each([ [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { - const isSqs = name === "sqs" - const isLucene = name === "lucene" const isInMemory = name === "in-memory" - const isInternal = isSqs || isLucene || isInMemory + const isInternal = !dsProvider const isOracle = name === DatabaseName.ORACLE - const isSql = !isInMemory && !isLucene + const isSql = !isInMemory const config = setup.getConfig() - let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let client: Knex | undefined let tableOrViewId: string @@ -111,12 +106,7 @@ describe.each([ } beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: true }, () => - config.init() - ) - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) + await config.init() if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { @@ -140,9 +130,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) async function createTable(schema?: TableSchema) { @@ -221,11 +208,6 @@ describe.each([ ])("from %s", (sourceType, createTableOrView) => { const isView = sourceType === "view" - if (isView && isLucene) { - // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests - return - } - class SearchAssertion { constructor(private readonly query: SearchRowRequest) {} @@ -598,19 +580,18 @@ describe.each([ ]) }) - !isLucene && - it("should return all rows matching the session user firstname when logical operator used", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { name: "{{ [user].firstName }}" } }], - }, - }).toContainExactly([ - { - name: config.getUser().firstName, - appointment: future.toISOString(), - }, - ]) - }) + it("should return all rows matching the session user firstname when logical operator used", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { name: "{{ [user].firstName }}" } }], + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + ]) + }) it("should parse the date binding and return all rows after the resolved value", async () => { await tk.withFreeze(serverTime, async () => { @@ -1034,21 +1015,19 @@ describe.each([ }).toFindNothing() }) - !isLucene && - it("ignores low if it's an empty object", async () => { - await expectQuery({ - // @ts-ignore - range: { name: { low: {}, high: "z" } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) + it("ignores low if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: {}, high: "z" } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - !isLucene && - it("ignores high if it's an empty object", async () => { - await expectQuery({ - // @ts-ignore - range: { name: { low: "a", high: {} } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) + it("ignores high if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: "a", high: {} } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("empty", () => { @@ -1202,31 +1181,23 @@ describe.each([ await expectQuery({ oneOf: { age: [2] } }).toFindNothing() }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can convert from a string", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - age: "1", - }, - }).toContainExactly([{ age: 1 }]) - }) + it("can convert from a string", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1", + }, + }).toContainExactly([{ age: 1 }]) + }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can find multiple values for same column", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - age: "1,10", - }, - }).toContainExactly([{ age: 1 }, { age: 10 }]) - }) + it("can find multiple values for same column", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1,10", + }, + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) }) describe("range", () => { @@ -1648,7 +1619,8 @@ describe.each([ }) }) - isSqs && + isInternal && + !isInMemory && describe("AI Column", () => { const UNEXISTING_AI_COLUMN = "Real LLM Response" @@ -1879,47 +1851,43 @@ describe.each([ }) }) - // Range searches against bigints don't seem to work at all in Lucene, and I - // couldn't figure out why. Given that we're replacing Lucene with SQS, - // we've decided not to spend time on it. - !isLucene && - describe("range", () => { - it("successfully finds a row", async () => { - await expectQuery({ - range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }]) - }) - - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) - }) - - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { num: { low: "5", high: "5" } }, - }).toFindNothing() - }) - - it("can search using just a low value", async () => { - await expectQuery({ - range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("can search using just a high value", async () => { - await expectQuery({ - range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) - }) + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }]) }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing() + }) + + it("can search using just a low value", async () => { + await expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("can search using just a high value", async () => { + await expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + }) }) isInternal && @@ -2016,94 +1984,93 @@ describe.each([ }).toFindNothing() }) - isSqs && - it("can search using just a low value", async () => { - await expectQuery({ - range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }]) - }) + it("can search using just a low value", async () => { + await expectQuery({ + range: { auto: { low: 9 } }, + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) - isSqs && - it("can search using just a high value", async () => { - await expectQuery({ - range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }]) - }) + it("can search using just a high value", async () => { + await expectQuery({ + range: { auto: { high: 2 } }, + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) }) - isSqs && - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([ - { auto: 1 }, - { auto: 2 }, - { auto: 3 }, - { auto: 4 }, - { auto: 5 }, - { auto: 6 }, - { auto: 7 }, - { auto: 8 }, - { auto: 9 }, - { auto: 10 }, - ]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([ - { auto: 10 }, - { auto: 9 }, - { auto: 8 }, - { auto: 7 }, - { auto: 6 }, - { auto: 5 }, - { auto: 4 }, - { auto: 3 }, - { auto: 2 }, - { auto: 1 }, - ]) - }) - - // This is important for pagination. The order of results must always - // be stable or pagination will break. We don't want the user to need - // to specify an order for pagination to work. - it("is stable without a sort specified", async () => { - let { rows: fullRowList } = await config.api.row.search( - tableOrViewId, - { - tableId: tableOrViewId, - query: {}, - } - ) - - // repeat the search many times to check the first row is always the same - let bookmark: string | number | undefined, - hasNextPage: boolean | undefined = true, - rowCount: number = 0 - do { - const response = await config.api.row.search(tableOrViewId, { - tableId: tableOrViewId, - limit: 1, - paginate: true, - query: {}, - bookmark, - }) - bookmark = response.bookmark - hasNextPage = response.hasNextPage - expect(response.rows.length).toEqual(1) - const foundRow = response.rows[0] - expect(foundRow).toEqual(fullRowList[rowCount++]) - } while (hasNextPage) - }) + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.ASCENDING, + sortType: SortType.NUMBER, + }).toMatchExactly([ + { auto: 1 }, + { auto: 2 }, + { auto: 3 }, + { auto: 4 }, + { auto: 5 }, + { auto: 6 }, + { auto: 7 }, + { auto: 8 }, + { auto: 9 }, + { auto: 10 }, + ]) }) + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.DESCENDING, + sortType: SortType.NUMBER, + }).toMatchExactly([ + { auto: 10 }, + { auto: 9 }, + { auto: 8 }, + { auto: 7 }, + { auto: 6 }, + { auto: 5 }, + { auto: 4 }, + { auto: 3 }, + { auto: 2 }, + { auto: 1 }, + ]) + }) + + // This is important for pagination. The order of results must always + // be stable or pagination will break. We don't want the user to need + // to specify an order for pagination to work. + it("is stable without a sort specified", async () => { + let { rows: fullRowList } = await config.api.row.search( + tableOrViewId, + { + tableId: tableOrViewId, + query: {}, + } + ) + + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { + const response = await config.api.row.search(tableOrViewId, { + tableId: tableOrViewId, + limit: 1, + paginate: true, + query: {}, + bookmark, + }) + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) + }) + }) + describe("pagination", () => { it("should paginate through all rows", async () => { // @ts-ignore @@ -2392,11 +2359,9 @@ describe.each([ }) }) - // This will never work for Lucene. - !isLucene && - // It also can't work for in-memory searching because the related table name - // isn't available. - !isInMemory && + // It also can't work for in-memory searching because the related table name + // isn't available. + !isInMemory && describe.each([ RelationshipType.ONE_TO_MANY, RelationshipType.MANY_TO_ONE, @@ -2847,42 +2812,40 @@ describe.each([ }) }) - // lucene can't count the total rows - !isLucene && - describe("row counting", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - await createRows([{ name: "a" }, { name: "b" }]) - }) - - it("should be able to count rows when option set", async () => { - await expectSearch({ - countRows: true, - query: { - notEmpty: { - name: true, - }, - }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) }) - }) - - it("shouldn't count rows when option is not set", async () => { - await expectSearch({ - countRows: false, - query: { - notEmpty: { - name: true, - }, - }, - }).toNotHaveProperty(["totalRows"]) + describe("row counting", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + name: { + name: "name", + type: FieldType.STRING, + }, }) + await createRows([{ name: "a" }, { name: "b" }]) }) + it("should be able to count rows when option set", async () => { + await expectSearch({ + countRows: true, + query: { + notEmpty: { + name: true, + }, + }, + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) + + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ + countRows: false, + query: { + notEmpty: { + name: true, + }, + }, + }).toNotHaveProperty(["totalRows"]) + }) + }) + describe("Invalid column definitions", () => { beforeAll(async () => { // need to create an invalid table - means ignoring typescript @@ -3065,9 +3028,7 @@ describe.each([ }) }) - // This was never actually supported in Lucene but SQS does support it, so may - // as well have a test for it. - ;(isSqs || isInMemory) && + isInternal && describe("space at start of column name", () => { beforeAll(async () => { tableOrViewId = await createTableOrView({ @@ -3100,7 +3061,7 @@ describe.each([ }) }) - isSqs && + isInternal && !isView && describe("duplicate columns", () => { beforeAll(async () => { @@ -3262,291 +3223,286 @@ describe.each([ }) }) - !isLucene && - describe("$and", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) + describe("$and", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) - }) + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) - }) + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Ja" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) - }) + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Ja" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - range: { age: { low: 1, high: 10 } }, - }, - { string: { name: "Ja" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([{ age: 1, name: "Jane" }]) - }) - - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toFindNothing() - }) - - !isInMemory && - it("validates conditions that are not objects", async () => { - await expect( - expectQuery({ + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { $and: { conditions: [ - { equal: { age: 10 } }, - "invalidCondition" as any, - ], - }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1]" must be of type object' - ) - }) - - !isInMemory && - it("validates $and without conditions", async () => { - await expect( - expectQuery({ - $and: { - conditions: [ - { equal: { age: 10 } }, { - $and: { - conditions: undefined as any, - }, + range: { age: { low: 1, high: 10 } }, }, + { string: { name: "Ja" } }, ], }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' - ) - }) + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) - // onEmptyFilter cannot be sent to view searches - !isView && - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $and: { - conditions: [{ equal: { name: "" } }], - }, + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toFindNothing() + }) + + !isInMemory && + it("validates conditions that are not objects", async () => { + await expect( + expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], }, }).toFindNothing() - }) + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1]" must be of type object' + ) + }) - it("returns all rows when onEmptyFilter set to all", async () => { + !isInMemory && + it("validates $and without conditions", async () => { + await expect( + expectQuery({ + $and: { + conditions: [ + { equal: { age: 10 } }, + { + $and: { + conditions: undefined as any, + }, + }, + ], + }, + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' + ) + }) + + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { await expectSearch({ query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, $and: { conditions: [{ equal: { name: "" } }], }, }, - }).toHaveLength(4) - }) - }) - - !isLucene && - describe("$or", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Jan" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - range: { age: { low: 1, high: 7 } }, - }, - { string: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], - }, }).toFindNothing() }) - it("can nest $and under $or filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $and: { - conditions: [ - { - range: { age: { low: 1, high: 8 } }, - }, - { equal: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("can nest $or under $and filters", async () => { - await expectQuery({ + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, $and: { - conditions: [ - { - $or: { - conditions: [ - { - range: { age: { low: 1, high: 8 } }, - }, - { equal: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], + conditions: [{ equal: { name: "" } }], }, - }).toContainExactly([{ age: 1, name: "Jane" }]) + }, + }).toHaveLength(4) + }) + }) + + describe("$or", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - // onEmptyFilter cannot be sent to view searches - !isView && - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Jan" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { $or: { - conditions: [{ equal: { name: "" } }], + conditions: [ + { + range: { age: { low: 1, high: 7 } }, + }, + { string: { name: "Jan" } }, + ], }, + equal: { name: "Jane" }, }, - }).toFindNothing() - }) + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - it("returns all rows when onEmptyFilter set to all", async () => { + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], + }, + }).toFindNothing() + }) + + it("can nest $and under $or filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("can nest $or under $and filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) + + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { await expectSearch({ query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, $or: { conditions: [{ equal: { name: "" } }], }, }, - }).toHaveLength(4) + }).toFindNothing() }) + + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $or: { + conditions: [{ equal: { name: "" } }], + }, + }, + }).toHaveLength(4) }) + }) isSql && describe("max related columns", () => { @@ -3590,8 +3546,7 @@ describe.each([ }) }) - isSql && - !isSqs && + !isInternal && describe("SQL injection", () => { const badStrings = [ "1; DROP TABLE %table_name%;", diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index d5483c54b4..725938cb04 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -2,7 +2,6 @@ import * as setup from "./utilities" import path from "path" import nock from "nock" import { generator } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" interface App { background: string @@ -82,48 +81,36 @@ describe("/templates", () => { }) describe("create app from template", () => { - it.each(["sqs", "lucene"])( - `should be able to create an app from a template (%s)`, - async source => { - await features.testutils.withFeatureFlags( - "*", - { SQS: source === "sqs" }, - async () => { - const name = generator.guid().replaceAll("-", "") - const url = `/${name}` + it("should be able to create an app from a template", async () => { + const name = generator.guid().replaceAll("-", "") + const url = `/${name}` - const app = await config.api.application.create({ - name, - url, - useTemplate: "true", - templateName: "Agency Client Portal", - templateKey: "app/agency-client-portal", - }) - expect(app.name).toBe(name) - expect(app.url).toBe(url) + const app = await config.api.application.create({ + name, + url, + useTemplate: "true", + templateName: "Agency Client Portal", + templateKey: "app/agency-client-portal", + }) + expect(app.name).toBe(name) + expect(app.url).toBe(url) - await config.withApp(app, async () => { - const tables = await config.api.table.fetch() - expect(tables).toHaveLength(2) + await config.withApp(app, async () => { + const tables = await config.api.table.fetch() + expect(tables).toHaveLength(2) - tables.sort((a, b) => a.name.localeCompare(b.name)) - const [agencyProjects, users] = tables - expect(agencyProjects.name).toBe("Agency Projects") - expect(users.name).toBe("Users") + tables.sort((a, b) => a.name.localeCompare(b.name)) + const [agencyProjects, users] = tables + expect(agencyProjects.name).toBe("Agency Projects") + expect(users.name).toBe("Users") - const { rows } = await config.api.row.search( - agencyProjects._id!, - { - tableId: agencyProjects._id!, - query: {}, - } - ) + const { rows } = await config.api.row.search(agencyProjects._id!, { + tableId: agencyProjects._id!, + query: {}, + }) - expect(rows).toHaveLength(3) - }) - } - ) - } - ) + expect(rows).toHaveLength(3) + }) + }) }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 415b22d407..652fb92b6f 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -40,10 +40,9 @@ import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import merge from "lodash/merge" import { quotas } from "@budibase/pro" -import { db, roles, features, context } from "@budibase/backend-core" +import { db, roles, context } from "@budibase/backend-core" describe.each([ - ["lucene", undefined], ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], @@ -52,14 +51,11 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() - const isSqs = name === "sqs" - const isLucene = name === "lucene" - const isInternal = isSqs || isLucene + const isInternal = name === "sqs" let table: Table let rawDatasource: Datasource | undefined let datasource: Datasource | undefined - let envCleanup: (() => void) | undefined function saveTableRequest( ...overrides: Partial>[] @@ -106,13 +102,7 @@ describe.each([ } beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => - config.init() - ) - - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) + await config.init() if (dsProvider) { rawDatasource = await dsProvider @@ -125,9 +115,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) beforeEach(() => { @@ -855,41 +842,40 @@ describe.each([ }) }) - !isLucene && - it("does not get confused when a calculation field shadows a basic one", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - age: { - name: "age", - type: FieldType.NUMBER, - }, - }, - }) - ) - - await config.api.row.bulkImport(table._id!, { - rows: [{ age: 1 }, { age: 2 }, { age: 3 }], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, + it("does not get confused when a calculation field shadows a basic one", async () => { + const table = await config.api.table.save( + saveTableRequest({ schema: { age: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", + name: "age", + type: FieldType.NUMBER, }, }, }) + ) - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].age).toEqual(6) + await config.api.row.bulkImport(table._id!, { + rows: [{ age: 1 }, { age: 2 }, { age: 3 }], }) + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + age: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].age).toEqual(6) + }) + // We don't allow the creation of tables with most JsonTypes when using // external datasources. isInternal && @@ -1453,206 +1439,205 @@ describe.each([ ) }) - !isLucene && - describe("calculation views", () => { - let table: Table - let view: ViewV2 + describe("calculation views", () => { + let table: Table + let view: ViewV2 - beforeEach(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - country: { - name: "country", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, + beforeEach(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, }, }, - }) - ) - - view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { country: { - visible: true, + name: "country", + type: FieldType.STRING, }, age: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", + name: "age", + type: FieldType.NUMBER, }, }, }) + ) - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "Steve", - age: 30, - country: "UK", - }, - { - name: "Jane", - age: 31, - country: "UK", - }, - { - name: "Ruari", - age: 32, - country: "USA", - }, - { - name: "Alice", - age: 33, - country: "USA", - }, - ], - }) + view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + country: { + visible: true, + }, + age: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, }) - it("returns the expected rows prior to modification", async () => { - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - }, - { - country: "UK", - age: 61, - }, - ]) - ) - }) - - it("can remove a group by field", async () => { - delete view.schema!.country - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows).toEqual( - expect.arrayContaining([ - { - age: 126, - }, - ]) - ) - }) - - it("can remove a calculation field", async () => { - delete view.schema!.age - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(4) - - // Because the removal of the calculation field actually makes this - // no longer a calculation view, these rows will now have _id and - // _rev fields. - expect(rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ country: "UK" }), - expect.objectContaining({ country: "UK" }), - expect.objectContaining({ country: "USA" }), - expect.objectContaining({ country: "USA" }), - ]) - ) - }) - - it("can add a new group by field", async () => { - view.schema!.name = { visible: true } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(4) - expect(rows).toEqual( - expect.arrayContaining([ - { - name: "Steve", - age: 30, - country: "UK", - }, - { - name: "Jane", - age: 31, - country: "UK", - }, - { - name: "Ruari", - age: 32, - country: "USA", - }, - { - name: "Alice", - age: 33, - country: "USA", - }, - ]) - ) - }) - - it("can add a new group by field that is invisible, even if required on the table", async () => { - view.schema!.name = { visible: false } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - }, - { - country: "UK", - age: 61, - }, - ]) - ) - }) - - it("can add a new calculation field", async () => { - view.schema!.count = { - visible: true, - calculationType: CalculationType.COUNT, - field: "age", - } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - count: 2, - }, - { - country: "UK", - age: 61, - count: 2, - }, - ]) - ) + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Steve", + age: 30, + country: "UK", + }, + { + name: "Jane", + age: 31, + country: "UK", + }, + { + name: "Ruari", + age: 32, + country: "USA", + }, + { + name: "Alice", + age: 33, + country: "USA", + }, + ], }) }) + + it("returns the expected rows prior to modification", async () => { + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + }, + { + country: "UK", + age: 61, + }, + ]) + ) + }) + + it("can remove a group by field", async () => { + delete view.schema!.country + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows).toEqual( + expect.arrayContaining([ + { + age: 126, + }, + ]) + ) + }) + + it("can remove a calculation field", async () => { + delete view.schema!.age + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(4) + + // Because the removal of the calculation field actually makes this + // no longer a calculation view, these rows will now have _id and + // _rev fields. + expect(rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ country: "UK" }), + expect.objectContaining({ country: "UK" }), + expect.objectContaining({ country: "USA" }), + expect.objectContaining({ country: "USA" }), + ]) + ) + }) + + it("can add a new group by field", async () => { + view.schema!.name = { visible: true } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(4) + expect(rows).toEqual( + expect.arrayContaining([ + { + name: "Steve", + age: 30, + country: "UK", + }, + { + name: "Jane", + age: 31, + country: "UK", + }, + { + name: "Ruari", + age: 32, + country: "USA", + }, + { + name: "Alice", + age: 33, + country: "USA", + }, + ]) + ) + }) + + it("can add a new group by field that is invisible, even if required on the table", async () => { + view.schema!.name = { visible: false } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + }, + { + country: "UK", + age: 61, + }, + ]) + ) + }) + + it("can add a new calculation field", async () => { + view.schema!.count = { + visible: true, + calculationType: CalculationType.COUNT, + field: "age", + } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + count: 2, + }, + { + country: "UK", + age: 61, + count: 2, + }, + ]) + ) + }) + }) }) describe("delete", () => { @@ -2293,25 +2278,75 @@ describe.each([ }) }) - !isLucene && - describe("calculation views", () => { - it("should not remove calculation columns when modifying table schema", async () => { - let table = await config.api.table.save( + describe("calculation views", () => { + it("should not remove calculation columns when modifying table schema", async () => { + let table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + let view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + table = await config.api.table.get(table._id!) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + name: { + name: "name", + type: FieldType.STRING, + constraints: { presence: true }, + }, + }, + }) + + view = await config.api.viewV2.get(view.id) + expect(Object.keys(view.schema!).sort()).toEqual([ + "age", + "id", + "name", + "sum", + ]) + }) + + describe("bigints", () => { + let table: Table + let view: ViewV2 + + beforeEach(async () => { + table = await config.api.table.save( saveTableRequest({ schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, + bigint: { + name: "bigint", + type: FieldType.BIGINT, }, }, }) ) - let view = await config.api.viewV2.create({ + view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), type: ViewV2Type.CALCULATION, @@ -2319,92 +2354,41 @@ describe.each([ sum: { visible: true, calculationType: CalculationType.SUM, - field: "age", + field: "bigint", }, }, }) - - table = await config.api.table.get(table._id!) - await config.api.table.save({ - ...table, - schema: { - ...table.schema, - name: { - name: "name", - type: FieldType.STRING, - constraints: { presence: true }, - }, - }, - }) - - view = await config.api.viewV2.get(view.id) - expect(Object.keys(view.schema!).sort()).toEqual([ - "age", - "id", - "name", - "sum", - ]) }) - describe("bigints", () => { - let table: Table - let view: ViewV2 - - beforeEach(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - bigint: { - name: "bigint", - type: FieldType.BIGINT, - }, - }, - }) - ) - - view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "bigint", - }, - }, - }) + it("should not lose precision handling ints larger than JSs int53", async () => { + // The sum of the following 3 numbers cannot be represented by + // JavaScripts default int53 datatype for numbers, so this is a test + // that makes sure we aren't losing precision between the DB and the + // user. + await config.api.row.bulkImport(table._id!, { + rows: [ + { bigint: "1000000000000000000" }, + { bigint: "123" }, + { bigint: "321" }, + ], }) - it("should not lose precision handling ints larger than JSs int53", async () => { - // The sum of the following 3 numbers cannot be represented by - // JavaScripts default int53 datatype for numbers, so this is a test - // that makes sure we aren't losing precision between the DB and the - // user. - await config.api.row.bulkImport(table._id!, { - rows: [ - { bigint: "1000000000000000000" }, - { bigint: "123" }, - { bigint: "321" }, - ], - }) + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual("1000000000000000444") + }) - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual("1000000000000000444") + it("should be able to handle up to 2**63 - 1 bigints", async () => { + await config.api.row.bulkImport(table._id!, { + rows: [{ bigint: "9223372036854775806" }, { bigint: "1" }], }) - it("should be able to handle up to 2**63 - 1 bigints", async () => { - await config.api.row.bulkImport(table._id!, { - rows: [{ bigint: "9223372036854775806" }, { bigint: "1" }], - }) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual("9223372036854775807") - }) + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual("9223372036854775807") }) }) + }) }) describe("row operations", () => { @@ -2721,440 +2705,487 @@ describe.each([ }) }) - !isLucene && - describe("search", () => { - it("returns empty rows from view when no schema is passed", async () => { - const rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, {}) - ) - ) - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(10) - expect(response).toEqual({ - rows: expect.arrayContaining( - rows.map(r => ({ - _viewId: view.id, - tableId: table._id, - id: r.id, - _id: r._id, - _rev: r._rev, - ...(isInternal - ? { - type: "row", - updatedAt: expect.any(String), - createdAt: expect.any(String), - } - : {}), - })) - ), - ...(isInternal - ? {} - : { - hasNextPage: false, - }), - }) + describe("search", () => { + it("returns empty rows from view when no schema is passed", async () => { + const rows = await Promise.all( + Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) + ) + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(10) + expect(response).toEqual({ + rows: expect.arrayContaining( + rows.map(r => ({ + _viewId: view.id, + tableId: table._id, + id: r.id, + _id: r._id, + _rev: r._rev, + ...(isInternal + ? { + type: "row", + updatedAt: expect.any(String), + createdAt: expect.any(String), + } + : {}), + })) + ), + ...(isInternal + ? {} + : { + hasNextPage: false, + }), + }) + }) + + it("searching respects the view filters", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", }) - it("searching respects the view filters", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - }, - ], - }, - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(1) - expect(response).toEqual({ - rows: expect.arrayContaining([ + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ { - _viewId: view.id, - tableId: table._id, - id: two.id, - two: two.two, - _id: two._id, - _rev: two._rev, - ...(isInternal - ? { - type: "row", - createdAt: expect.any(String), - updatedAt: expect.any(String), - } - : {}), + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], }, - ]), - ...(isInternal - ? {} - : { - hasNextPage: false, - }), - }) - }) - - it("views filters are respected even if the column is hidden", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - }, - ], - }, - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: false }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: two._id }), - ]) - }) - - it("views without data can be returned", async () => { - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toHaveLength(0) - }) - - it("respects the limit parameter", async () => { - await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, {}) - ) - ) - const limit = generator.integer({ min: 1, max: 8 }) - const response = await config.api.viewV2.search(view.id, { - limit, - query: {}, - }) - expect(response.rows).toHaveLength(limit) - }) - - it("can handle pagination", async () => { - await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, {}) - ) - ) - const rows = (await config.api.viewV2.search(view.id)).rows - - const page1 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - query: {}, - countRows: true, - }) - expect(page1).toEqual({ - rows: expect.arrayContaining(rows.slice(0, 4)), - hasNextPage: true, - bookmark: expect.anything(), - totalRows: 10, - }) - - const page2 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - bookmark: page1.bookmark, - query: {}, - countRows: true, - }) - expect(page2).toEqual({ - rows: expect.arrayContaining(rows.slice(4, 8)), - hasNextPage: true, - bookmark: expect.anything(), - totalRows: 10, - }) - - const page3 = await config.api.viewV2.search(view.id, { - paginate: true, - limit: 4, - bookmark: page2.bookmark, - query: {}, - countRows: true, - }) - const expectation: SearchResponse = { - rows: expect.arrayContaining(rows.slice(8)), - hasNextPage: false, - totalRows: 10, - } - if (isLucene) { - expectation.bookmark = expect.anything() - } - expect(page3).toEqual(expectation) - }) - - const sortTestOptions: [ - { - field: string - order?: SortOrder - type?: SortType + ], }, - string[] - ][] = [ - [ - { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - }, - ["Alice", "Bob", "Charly", "Danny"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "name", - order: SortOrder.DESCENDING, - type: SortType.STRING, - }, - ["Danny", "Charly", "Bob", "Alice"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - type: SortType.NUMBER, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.ASCENDING, - }, - ["Danny", "Alice", "Charly", "Bob"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - [ - { - field: "age", - order: SortOrder.DESCENDING, - type: SortType.NUMBER, - }, - ["Bob", "Charly", "Alice", "Danny"], - ], - ] - - describe("sorting", () => { - let table: Table - const viewSchema = { + schema: { id: { visible: true }, - age: { visible: true }, - name: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response).toEqual({ + rows: expect.arrayContaining([ + { + _viewId: view.id, + tableId: table._id, + id: two.id, + two: two.two, + _id: two._id, + _rev: two._rev, + ...(isInternal + ? { + type: "row", + createdAt: expect.any(String), + updatedAt: expect.any(String), + } + : {}), + }, + ]), + ...(isInternal + ? {} + : { + hasNextPage: false, + }), + }) + }) + + it("views filters are respected even if the column is hidden", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + const two = await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + }, + ], + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: false }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: two._id }), + ]) + }) + + it("views without data can be returned", async () => { + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toHaveLength(0) + }) + + it("respects the limit parameter", async () => { + await Promise.all( + Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) + ) + const limit = generator.integer({ min: 1, max: 8 }) + const response = await config.api.viewV2.search(view.id, { + limit, + query: {}, + }) + expect(response.rows).toHaveLength(limit) + }) + + it("can handle pagination", async () => { + await Promise.all( + Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) + ) + const rows = (await config.api.viewV2.search(view.id)).rows + + const page1 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + query: {}, + countRows: true, + }) + expect(page1).toEqual({ + rows: expect.arrayContaining(rows.slice(0, 4)), + hasNextPage: true, + bookmark: expect.anything(), + totalRows: 10, + }) + + const page2 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + bookmark: page1.bookmark, + query: {}, + countRows: true, + }) + expect(page2).toEqual({ + rows: expect.arrayContaining(rows.slice(4, 8)), + hasNextPage: true, + bookmark: expect.anything(), + totalRows: 10, + }) + + const page3 = await config.api.viewV2.search(view.id, { + paginate: true, + limit: 4, + bookmark: page2.bookmark, + query: {}, + countRows: true, + }) + const expectation: SearchResponse = { + rows: expect.arrayContaining(rows.slice(8)), + hasNextPage: false, + totalRows: 10, + } + expect(page3).toEqual(expectation) + }) + + const sortTestOptions: [ + { + field: string + order?: SortOrder + type?: SortType + }, + string[] + ][] = [ + [ + { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + }, + ["Alice", "Bob", "Charly", "Danny"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "name", + order: SortOrder.DESCENDING, + type: SortType.STRING, + }, + ["Danny", "Charly", "Bob", "Alice"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + type: SortType.NUMBER, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.ASCENDING, + }, + ["Danny", "Alice", "Charly", "Bob"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + [ + { + field: "age", + order: SortOrder.DESCENDING, + type: SortType.NUMBER, + }, + ["Bob", "Charly", "Alice", "Danny"], + ], + ] + + describe("sorting", () => { + let table: Table + const viewSchema = { + id: { visible: true }, + age: { visible: true }, + name: { visible: true }, + } + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + }, + surname: { + type: FieldType.STRING, + name: "surname", + }, + age: { + type: FieldType.NUMBER, + name: "age", + }, + address: { + type: FieldType.STRING, + name: "address", + }, + jobTitle: { + type: FieldType.STRING, + name: "jobTitle", + }, + }, + }) + ) + + const users = [ + { name: "Alice", age: 25 }, + { name: "Bob", age: 30 }, + { name: "Charly", age: 27 }, + { name: "Danny", age: 15 }, + ] + await Promise.all( + users.map(u => + config.api.row.save(table._id!, { + tableId: table._id, + ...u, + }) + ) + ) + }) + + it.each(sortTestOptions)( + "allow sorting (%s)", + async (sortParams, expected) => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + sort: sortParams, + schema: viewSchema, + }) + + const response = await config.api.viewV2.search(view.id) + + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( + expected.map(name => expect.objectContaining({ name })) + ) } + ) - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - type: "table", - schema: { - name: { - type: FieldType.STRING, - name: "name", - }, - surname: { - type: FieldType.STRING, - name: "surname", - }, - age: { - type: FieldType.NUMBER, - name: "age", - }, - address: { - type: FieldType.STRING, - name: "address", - }, - jobTitle: { - type: FieldType.STRING, - name: "jobTitle", - }, - }, - }) + it.each(sortTestOptions)( + "allow override the default view sorting (%s)", + async (sortParams, expected) => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + sort: { + field: "name", + order: SortOrder.ASCENDING, + type: SortType.STRING, + }, + schema: viewSchema, + }) + + const response = await config.api.viewV2.search(view.id, { + sort: sortParams.field, + sortOrder: sortParams.order, + sortType: sortParams.type, + query: {}, + }) + + expect(response.rows).toHaveLength(4) + expect(response.rows).toEqual( + expected.map(name => expect.objectContaining({ name })) ) + } + ) + }) - const users = [ - { name: "Alice", age: 25 }, - { name: "Bob", age: 30 }, - { name: "Charly", age: 27 }, - { name: "Danny", age: 15 }, - ] - await Promise.all( - users.map(u => - config.api.row.save(table._id!, { - tableId: table._id, - ...u, - }) - ) - ) - }) - - it.each(sortTestOptions)( - "allow sorting (%s)", - async (sortParams, expected) => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - sort: sortParams, - schema: viewSchema, - }) - - const response = await config.api.viewV2.search(view.id) - - expect(response.rows).toHaveLength(4) - expect(response.rows).toEqual( - expected.map(name => expect.objectContaining({ name })) - ) - } - ) - - it.each(sortTestOptions)( - "allow override the default view sorting (%s)", - async (sortParams, expected) => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - sort: { - field: "name", - order: SortOrder.ASCENDING, - type: SortType.STRING, - }, - schema: viewSchema, - }) - - const response = await config.api.viewV2.search(view.id, { - sort: sortParams.field, - sortOrder: sortParams.order, - sortType: sortParams.type, - query: {}, - }) - - expect(response.rows).toHaveLength(4) - expect(response.rows).toEqual( - expected.map(name => expect.objectContaining({ name })) - ) - } - ) + it("can query on top of the view filters", async () => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + const three = await config.api.row.save(table._id!, { + one: "foo3", + two: "bar3", }) - it("can query on top of the view filters", async () => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - const three = await config.api.row.save(table._id!, { - one: "foo3", - two: "bar3", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.NOT_EQUAL, - field: "one", - value: "foo2", - }, - ], - }, - ], - }, - schema: { - id: { visible: true }, - one: { visible: true }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - [BasicOperator.EQUAL]: { - two: "bar3", + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.NOT_EQUAL, + field: "one", + value: "foo2", + }, + ], }, - [BasicOperator.NOT_EMPTY]: { - two: null, - }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ _id: three._id }), - ]) - ) + ], + }, + schema: { + id: { visible: true }, + one: { visible: true }, + two: { visible: true }, + }, }) - it("can query on top of the view filters (using or filters)", async () => { + const response = await config.api.viewV2.search(view.id, { + query: { + [BasicOperator.EQUAL]: { + two: "bar3", + }, + [BasicOperator.NOT_EMPTY]: { + two: null, + }, + }, + }) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([expect.objectContaining({ _id: three._id })]) + ) + }) + + it("can query on top of the view filters (using or filters)", async () => { + const one = await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + const three = await config.api.row.save(table._id!, { + one: "foo3", + two: "bar3", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.NOT_EQUAL, + field: "one", + value: "foo2", + }, + ], + }, + ], + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + allOr: true, + [BasicOperator.NOT_EQUAL]: { + two: "bar", + }, + [BasicOperator.NOT_EMPTY]: { + two: null, + }, + }, + }) + expect(response.rows).toHaveLength(2) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ _id: one._id }), + expect.objectContaining({ _id: three._id }), + ]) + ) + }) + + it.each([true, false])( + "can filter a view without a view filter", + async allOr => { const one = await config.api.row.save(table._id!, { one: "foo", two: "bar", @@ -3163,27 +3194,10 @@ describe.each([ one: "foo2", two: "bar2", }) - const three = await config.api.row.save(table._id!, { - one: "foo3", - two: "bar3", - }) const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.NOT_EQUAL, - field: "one", - value: "foo2", - }, - ], - }, - ], - }, schema: { id: { visible: true }, one: { visible: false }, @@ -3193,1423 +3207,1351 @@ describe.each([ const response = await config.api.viewV2.search(view.id, { query: { - allOr: true, - [BasicOperator.NOT_EQUAL]: { + allOr, + equal: { two: "bar", }, - [BasicOperator.NOT_EMPTY]: { - two: null, - }, }, }) - expect(response.rows).toHaveLength(2) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ _id: one._id }), - expect.objectContaining({ _id: three._id }), - ]) - ) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: one._id }), + ]) + } + ) + + it.each([true, false])("cannot bypass a view filter", async allOr => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", + }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", }) - !isLucene && - it.each([true, false])( - "can filter a view without a view filter", - async allOr => { - const one = await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: one._id }), - ]) - } - ) + ], + }, + ], + }, + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) - !isLucene && - it.each([true, false])("cannot bypass a view filter", async allOr => { - await config.api.row.save(table._id!, { - one: "foo", + const response = await config.api.viewV2.search(view.id, { + query: { + allOr, + equal: { two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - }, - ], - }, - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", - }, - }, - }) - expect(response.rows).toHaveLength(0) - }) - - describe("foreign relationship columns", () => { - let envCleanup: () => void - beforeAll(() => { - envCleanup = features.testutils.setFeatureFlags("*", { - ENRICHED_RELATIONSHIPS: true, - }) - }) - - afterAll(() => { - envCleanup?.() - }) - - const createMainTable = async ( - links: { - name: string - tableId: string - fk: string - }[] - ) => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { title: { name: "title", type: FieldType.STRING } }, - }) - ) - await config.api.table.save({ - ...table, - schema: { - ...table.schema, - ...links.reduce((acc, c) => { - acc[c.name] = { - name: c.name, - relationshipType: RelationshipType.ONE_TO_MANY, - type: FieldType.LINK, - tableId: c.tableId, - fieldName: c.fk, - constraints: { type: "array" }, - } - return acc - }, {}), - }, - }) - return table - } - const createAuxTable = (schema: TableSchema) => - config.api.table.save( - saveTableRequest({ - primaryDisplay: "name", - schema: { - ...schema, - name: { name: "name", type: FieldType.STRING }, - }, - }) - ) - - it("returns squashed fields respecting the view config", async () => { - const auxTable = await createAuxTable({ - age: { name: "age", type: FieldType.NUMBER }, - }) - const auxRow = await config.api.row.save(auxTable._id!, { - name: generator.name(), - age: generator.age(), - }) - - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - await config.api.row.save(table._id!, { - title: generator.word(), - aux: [auxRow], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - title: { visible: true }, - aux: { - visible: true, - columns: { - name: { visible: false, readonly: false }, - age: { visible: true, readonly: true }, - }, - }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - expect(response.rows).toEqual([ - expect.objectContaining({ - aux: [ - { - _id: auxRow._id, - primaryDisplay: auxRow.name, - age: auxRow.age, - }, - ], - }), - ]) - }) - - it("enriches squashed fields", async () => { - const auxTable = await createAuxTable({ - user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - constraints: { presence: true }, - }, - }) - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - - const user = config.getUser() - const auxRow = await config.api.row.save(auxTable._id!, { - name: generator.name(), - user: user._id, - }) - await config.api.row.save(table._id!, { - title: generator.word(), - aux: [auxRow], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - title: { visible: true }, - aux: { - visible: true, - columns: { - name: { visible: true, readonly: true }, - user: { visible: true, readonly: true }, - }, - }, - }, - }) - - const response = await config.api.viewV2.search(view.id) - - expect(response.rows).toEqual([ - expect.objectContaining({ - aux: [ - { - _id: auxRow._id, - primaryDisplay: auxRow.name, - name: auxRow.name, - user: { - _id: user._id, - email: user.email, - firstName: user.firstName, - lastName: user.lastName, - primaryDisplay: user.email, - }, - }, - ], - }), - ]) - }) + }, + }, }) + expect(response.rows).toHaveLength(0) + }) - !isLucene && - describe("calculations", () => { - let table: Table - let rows: Row[] - - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, { - quantity: generator.natural({ min: 1, max: 10 }), - price: generator.natural({ min: 1, max: 10 }), - }) - ) - ) - }) - - it("should be able to search by calculations", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - type: ViewV2Type.CALCULATION, - name: generator.guid(), - schema: { - "Quantity Sum": { - visible: true, - calculationType: CalculationType.SUM, - field: "quantity", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - "Quantity Sum": rows.reduce( - (acc, r) => acc + r.quantity, - 0 - ), - }), - ]) - ) - - // Calculation views do not return rows that can be linked back to - // the source table, and so should not have an _id field. - for (const row of response.rows) { - expect("_id" in row).toBe(false) - } - }) - - it("should be able to group by a basic field", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { - visible: true, - field: "quantity", - }, - "Total Price": { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - const priceByQuantity: Record = {} - for (const row of rows) { - priceByQuantity[row.quantity] ??= 0 - priceByQuantity[row.quantity] += row.price - } - - for (const row of response.rows) { - expect(row["Total Price"]).toEqual( - priceByQuantity[row.quantity] - ) - } - }) - - it.each([ - CalculationType.COUNT, - CalculationType.SUM, - CalculationType.AVG, - CalculationType.MIN, - CalculationType.MAX, - ])("should be able to calculate $type", async type => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - aggregate: { - visible: true, - calculationType: type, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - function calculate( - type: CalculationType, - numbers: number[] - ): number { - switch (type) { - case CalculationType.COUNT: - return numbers.length - case CalculationType.SUM: - return numbers.reduce((a, b) => a + b, 0) - case CalculationType.AVG: - return numbers.reduce((a, b) => a + b, 0) / numbers.length - case CalculationType.MIN: - return Math.min(...numbers) - case CalculationType.MAX: - return Math.max(...numbers) - } - } - - const prices = rows.map(row => row.price) - const expected = calculate(type, prices) - const actual = response.rows[0].aggregate - - if (type === CalculationType.AVG) { - // The average calculation can introduce floating point rounding - // errors, so we need to compare to within a small margin of - // error. - expect(actual).toBeCloseTo(expected) - } else { - expect(actual).toEqual(expected) - } - }) - - it("should be able to do a COUNT(DISTINCT)", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "name", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "John", - }, - { - name: "John", - }, - { - name: "Sue", - }, - ], - }) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].count).toEqual(2) - }) - - it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { - await config.api.viewV2.create( - { - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "does not exist oh no", - }, - }, - }, - { - status: 400, - body: { - message: - 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', - }, - } - ) - }) - - it("should be able to filter on relationships", async () => { - const companies = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) - - const employees = await config.api.table.save( - saveTableRequest({ - schema: { - age: { - type: FieldType.NUMBER, - name: "age", - }, - name: { - type: FieldType.STRING, - name: "name", - }, - company: { - type: FieldType.LINK, - name: "company", - tableId: companies._id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "company", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: employees._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.EQUAL, - field: "company.name", - value: "Aperture Science Laboratories", - }, - ], - }, - ], - }, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", - }, - }, - }) - - const apertureScience = await config.api.row.save( - companies._id!, - { - name: "Aperture Science Laboratories", - } - ) - - const blackMesa = await config.api.row.save(companies._id!, { - name: "Black Mesa", - }) - - await Promise.all([ - config.api.row.save(employees._id!, { - name: "Alice", - age: 25, - company: apertureScience._id, - }), - config.api.row.save(employees._id!, { - name: "Bob", - age: 30, - company: apertureScience._id, - }), - config.api.row.save(employees._id!, { - name: "Charly", - age: 27, - company: blackMesa._id, - }), - config.api.row.save(employees._id!, { - name: "Danny", - age: 15, - company: blackMesa._id, - }), - ]) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(55) - }) - - it("should be able to count non-numeric fields", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - firstName: { - type: FieldType.STRING, - name: "firstName", - }, - lastName: { - type: FieldType.STRING, - name: "lastName", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - field: "firstName", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { firstName: "Jane", lastName: "Smith" }, - { firstName: "Jane", lastName: "Doe" }, - { firstName: "Alice", lastName: "Smith" }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(rows).toHaveLength(1) - expect(rows[0].count).toEqual(3) - }) - - it("should be able to filter rows on the view itself", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - queryUI: { - groups: [ - { - filters: [ - { - operator: BasicOperator.EQUAL, - field: "quantity", - value: 1, - }, - ], - }, - ], - }, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - }) - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to filter on group by fields", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: { - equal: { - quantity: 1, - }, - }, - }) - - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to sort by group by field", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "quantity", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - - it("should be able to sort by a calculation", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "sum", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - }) - - !isLucene && - it("should not need required fields to be present", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, - }, - }, - }) - ) - - await Promise.all([ - config.api.row.save(table._id!, { name: "Steve", age: 30 }), - config.api.row.save(table._id!, { name: "Jane", age: 31 }), - ]) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows[0].sum).toEqual(61) - }) - - it("should be able to filter on a single user field in both the view query and search query", async () => { + describe("foreign relationship columns", () => { + const createMainTable = async ( + links: { + name: string + tableId: string + fk: string + }[] + ) => { const table = await config.api.table.save( saveTableRequest({ + schema: { title: { name: "title", type: FieldType.STRING } }, + }) + ) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + ...links.reduce((acc, c) => { + acc[c.name] = { + name: c.name, + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: c.tableId, + fieldName: c.fk, + constraints: { type: "array" }, + } + return acc + }, {}), + }, + }) + return table + } + const createAuxTable = (schema: TableSchema) => + config.api.table.save( + saveTableRequest({ + primaryDisplay: "name", schema: { - user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, + ...schema, + name: { name: "name", type: FieldType.STRING }, }, }) ) + it("returns squashed fields respecting the view config", async () => { + const auxTable = await createAuxTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + age: generator.age(), + }) + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) await config.api.row.save(table._id!, { - user: config.getUser()._id, + title: generator.word(), + aux: [auxRow], }) const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: false, readonly: false }, + age: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + age: auxRow.age, + }, + ], + }), + ]) + }) + + it("enriches squashed fields", async () => { + const auxTable = await createAuxTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + constraints: { presence: true }, + }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const user = config.getUser() + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + user: user._id, + }) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: true, readonly: true }, + user: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await config.api.viewV2.search(view.id) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + name: auxRow.name, + user: { + _id: user._id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + primaryDisplay: user.email, + }, + }, + ], + }), + ]) + }) + }) + + describe("calculations", () => { + let table: Table + let rows: Row[] + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + type: ViewV2Type.CALCULATION, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) + ) + + // Calculation views do not return rows that can be linked back to + // the source table, and so should not have an _id field. + for (const row of response.rows) { + expect("_id" in row).toBe(false) + } + }) + + it("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { + visible: true, + field: "quantity", + }, + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + + for (const row of response.rows) { + expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) + } + }) + + it.each([ + CalculationType.COUNT, + CalculationType.SUM, + CalculationType.AVG, + CalculationType.MIN, + CalculationType.MAX, + ])("should be able to calculate $type", async type => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + aggregate: { + visible: true, + calculationType: type, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + function calculate(type: CalculationType, numbers: number[]): number { + switch (type) { + case CalculationType.COUNT: + return numbers.length + case CalculationType.SUM: + return numbers.reduce((a, b) => a + b, 0) + case CalculationType.AVG: + return numbers.reduce((a, b) => a + b, 0) / numbers.length + case CalculationType.MIN: + return Math.min(...numbers) + case CalculationType.MAX: + return Math.max(...numbers) + } + } + + const prices = rows.map(row => row.price) + const expected = calculate(type, prices) + const actual = response.rows[0].aggregate + + if (type === CalculationType.AVG) { + // The average calculation can introduce floating point rounding + // errors, so we need to compare to within a small margin of + // error. + expect(actual).toBeCloseTo(expected) + } else { + expect(actual).toEqual(expected) + } + }) + + it("should be able to do a COUNT(DISTINCT)", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "name", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "John", + }, + { + name: "John", + }, + { + name: "Sue", + }, + ], + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].count).toEqual(2) + }) + + it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "does not exist oh no", + }, + }, + }, + { + status: 400, + body: { + message: + 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', + }, + } + ) + }) + + it("should be able to filter on relationships", async () => { + const companies = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const employees = await config.api.table.save( + saveTableRequest({ + schema: { + age: { + type: FieldType.NUMBER, + name: "age", + }, + name: { + type: FieldType.STRING, + name: "name", + }, + company: { + type: FieldType.LINK, + name: "company", + tableId: companies._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "company", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: employees._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, queryUI: { groups: [ { filters: [ { operator: BasicOperator.EQUAL, - field: "user", - value: "{{ [user].[_id] }}", + field: "company.name", + value: "Aperture Science Laboratories", }, ], }, ], }, schema: { - user: { + sum: { visible: true, + calculationType: CalculationType.SUM, + field: "age", }, }, }) + const apertureScience = await config.api.row.save(companies._id!, { + name: "Aperture Science Laboratories", + }) + + const blackMesa = await config.api.row.save(companies._id!, { + name: "Black Mesa", + }) + + await Promise.all([ + config.api.row.save(employees._id!, { + name: "Alice", + age: 25, + company: apertureScience._id, + }), + config.api.row.save(employees._id!, { + name: "Bob", + age: 30, + company: apertureScience._id, + }), + config.api.row.save(employees._id!, { + name: "Charly", + age: 27, + company: blackMesa._id, + }), + config.api.row.save(employees._id!, { + name: "Danny", + age: 15, + company: blackMesa._id, + }), + ]) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(55) + }) + + it("should be able to count non-numeric fields", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + firstName: { + type: FieldType.STRING, + name: "firstName", + }, + lastName: { + type: FieldType.STRING, + name: "lastName", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + field: "firstName", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { firstName: "Jane", lastName: "Smith" }, + { firstName: "Jane", lastName: "Doe" }, + { firstName: "Alice", lastName: "Smith" }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].count).toEqual(3) + }) + + it("should be able to filter rows on the view itself", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "quantity", + value: 1, + }, + ], + }, + ], + }, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + }) + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(3) + }) + + it("should be able to filter on group by fields", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + const { rows } = await config.api.viewV2.search(view.id, { query: { equal: { - user: "{{ [user].[_id] }}", + quantity: 1, }, }, }) expect(rows).toHaveLength(1) - expect(rows[0].user._id).toEqual(config.getUser()._id) + expect(rows[0].sum).toEqual(3) }) - describe("search operators", () => { - let table: Table - beforeEach(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - string: { name: "string", type: FieldType.STRING }, - longform: { name: "longform", type: FieldType.LONGFORM }, - options: { - name: "options", - type: FieldType.OPTIONS, - constraints: { inclusion: ["a", "b", "c"] }, - }, - array: { - name: "array", - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - inclusion: ["a", "b", "c"], - }, - }, - number: { name: "number", type: FieldType.NUMBER }, - bigint: { name: "bigint", type: FieldType.BIGINT }, - datetime: { name: "datetime", type: FieldType.DATETIME }, - boolean: { name: "boolean", type: FieldType.BOOLEAN }, - user: { - name: "user", - type: FieldType.BB_REFERENCE_SINGLE, - subtype: BBReferenceFieldSubType.USER, - }, - users: { - name: "users", - type: FieldType.BB_REFERENCE, - subtype: BBReferenceFieldSubType.USER, - constraints: { - type: JsonFieldSubType.ARRAY, - }, - }, + it("should be able to sort by group by field", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", }, - }) - ) + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, }) - interface TestCase { - name: string - query: UISearchFilter | (() => UISearchFilter) - insert: Row[] | (() => Row[]) - expected: Row[] | (() => Row[]) - searchOpts?: Partial - } - - function simpleQuery(...filters: LegacyFilter[]): UISearchFilter { - return { groups: [{ filters }] } - } - - const testCases: TestCase[] = [ - { - name: "empty query return all", - insert: [{ string: "foo" }], - query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, }, - expected: [{ string: "foo" }], - }, - { - name: "empty query return none", - insert: [{ string: "foo" }], - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "quantity", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + + it("should be able to sort by a calculation", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", }, - expected: [], }, - { - name: "simple string search", - insert: [{ string: "foo" }], - query: simpleQuery({ + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "sum", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + }) + + it("should not need required fields to be present", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + await Promise.all([ + config.api.row.save(table._id!, { name: "Steve", age: 30 }), + config.api.row.save(table._id!, { name: "Jane", age: 31 }), + ]) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].sum).toEqual(61) + }) + + it("should be able to filter on a single user field in both the view query and search query", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + }, + }) + ) + + await config.api.row.save(table._id!, { + user: config.getUser()._id, + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: { + groups: [ + { + filters: [ + { + operator: BasicOperator.EQUAL, + field: "user", + value: "{{ [user].[_id] }}", + }, + ], + }, + ], + }, + schema: { + user: { + visible: true, + }, + }, + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: { + equal: { + user: "{{ [user].[_id] }}", + }, + }, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].user._id).toEqual(config.getUser()._id) + }) + + describe("search operators", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + string: { name: "string", type: FieldType.STRING }, + longform: { name: "longform", type: FieldType.LONGFORM }, + options: { + name: "options", + type: FieldType.OPTIONS, + constraints: { inclusion: ["a", "b", "c"] }, + }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["a", "b", "c"], + }, + }, + number: { name: "number", type: FieldType.NUMBER }, + bigint: { name: "bigint", type: FieldType.BIGINT }, + datetime: { name: "datetime", type: FieldType.DATETIME }, + boolean: { name: "boolean", type: FieldType.BOOLEAN }, + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + }, + users: { + name: "users", + type: FieldType.BB_REFERENCE, + subtype: BBReferenceFieldSubType.USER, + constraints: { + type: JsonFieldSubType.ARRAY, + }, + }, + }, + }) + ) + }) + + interface TestCase { + name: string + query: UISearchFilter | (() => UISearchFilter) + insert: Row[] | (() => Row[]) + expected: Row[] | (() => Row[]) + searchOpts?: Partial + } + + function simpleQuery(...filters: LegacyFilter[]): UISearchFilter { + return { groups: [{ filters }] } + } + + const testCases: TestCase[] = [ + { + name: "empty query return all", + insert: [{ string: "foo" }], + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + }, + expected: [{ string: "foo" }], + }, + { + name: "empty query return none", + insert: [{ string: "foo" }], + query: { + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + expected: [], + }, + { + name: "simple string search", + insert: [{ string: "foo" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }), + expected: [{ string: "foo" }], + }, + { + name: "non matching string search", + insert: [{ string: "foo" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "string", + value: "bar", + }), + expected: [], + }, + { + name: "allOr", + insert: [{ string: "bar" }, { string: "foo" }], + query: simpleQuery( + { operator: BasicOperator.EQUAL, field: "string", value: "foo", - }), - expected: [{ string: "foo" }], - }, - { - name: "non matching string search", - insert: [{ string: "foo" }], - query: simpleQuery({ + }, + { operator: BasicOperator.EQUAL, field: "string", value: "bar", - }), - expected: [], - }, - { - name: "allOr", - insert: [{ string: "bar" }, { string: "foo" }], - query: simpleQuery( - { - operator: BasicOperator.EQUAL, - field: "string", - value: "foo", - }, - { - operator: BasicOperator.EQUAL, - field: "string", - value: "bar", - }, - { - operator: "allOr", - } - ), - searchOpts: { - sort: "string", - sortOrder: SortOrder.ASCENDING, }, - expected: [{ string: "bar" }, { string: "foo" }], + { + operator: "allOr", + } + ), + searchOpts: { + sort: "string", + sortOrder: SortOrder.ASCENDING, }, - { - name: "can find rows with fuzzy search", - insert: [{ string: "foo" }, { string: "bar" }], - query: simpleQuery({ - operator: BasicOperator.FUZZY, - field: "string", - value: "fo", - }), - expected: [{ string: "foo" }], + expected: [{ string: "bar" }, { string: "foo" }], + }, + { + name: "can find rows with fuzzy search", + insert: [{ string: "foo" }, { string: "bar" }], + query: simpleQuery({ + operator: BasicOperator.FUZZY, + field: "string", + value: "fo", + }), + expected: [{ string: "foo" }], + }, + { + name: "can find nothing with fuzzy search", + insert: [{ string: "foo" }, { string: "bar" }], + query: simpleQuery({ + operator: BasicOperator.FUZZY, + field: "string", + value: "baz", + }), + expected: [], + }, + { + name: "can find numeric rows", + insert: [{ number: 1 }, { number: 2 }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "number", + value: 1, + }), + expected: [{ number: 1 }], + }, + { + name: "can find numeric values with rangeHigh", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery({ + operator: "rangeHigh", + field: "number", + value: 2, + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, }, - { - name: "can find nothing with fuzzy search", - insert: [{ string: "foo" }, { string: "bar" }], - query: simpleQuery({ - operator: BasicOperator.FUZZY, - field: "string", - value: "baz", - }), - expected: [], + expected: [{ number: 1 }, { number: 2 }], + }, + { + name: "can find numeric values with rangeLow", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery({ + operator: "rangeLow", + field: "number", + value: 2, + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, }, - { - name: "can find numeric rows", - insert: [{ number: 1 }, { number: 2 }], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "number", - value: 1, - }), - expected: [{ number: 1 }], - }, - { - name: "can find numeric values with rangeHigh", - insert: [{ number: 1 }, { number: 2 }, { number: 3 }], - query: simpleQuery({ + expected: [{ number: 2 }, { number: 3 }], + }, + { + name: "can find numeric values with full range", + insert: [{ number: 1 }, { number: 2 }, { number: 3 }], + query: simpleQuery( + { operator: "rangeHigh", field: "number", value: 2, - }), - searchOpts: { - sort: "number", - sortOrder: SortOrder.ASCENDING, }, - expected: [{ number: 1 }, { number: 2 }], - }, - { - name: "can find numeric values with rangeLow", - insert: [{ number: 1 }, { number: 2 }, { number: 3 }], - query: simpleQuery({ + { operator: "rangeLow", field: "number", value: 2, + } + ), + expected: [{ number: 2 }], + }, + { + name: "can find longform values", + insert: [{ longform: "foo" }, { longform: "bar" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "longform", + value: "foo", + }), + expected: [{ longform: "foo" }], + }, + { + name: "can find options values", + insert: [{ options: "a" }, { options: "b" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "options", + value: "a", + }), + expected: [{ options: "a" }], + }, + { + name: "can find array values", + insert: [ + // Number field here is just to guarantee order. + { number: 1, array: ["a"] }, + { number: 2, array: ["b"] }, + { number: 3, array: ["a", "c"] }, + ], + query: simpleQuery({ + operator: ArrayOperator.CONTAINS, + field: "array", + value: "a", + }), + searchOpts: { + sort: "number", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ array: ["a"] }, { array: ["a", "c"] }], + }, + { + name: "can find bigint values", + insert: [{ bigint: "1" }, { bigint: "2" }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "bigint", + type: FieldType.BIGINT, + value: "1", + }), + expected: [{ bigint: "1" }], + }, + { + name: "can find datetime values", + insert: [ + { datetime: "2021-01-01T00:00:00.000Z" }, + { datetime: "2021-01-02T00:00:00.000Z" }, + ], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "datetime", + type: FieldType.DATETIME, + value: "2021-01-01", + }), + expected: [{ datetime: "2021-01-01T00:00:00.000Z" }], + }, + { + name: "can find boolean values", + insert: [{ boolean: true }, { boolean: false }], + query: simpleQuery({ + operator: BasicOperator.EQUAL, + field: "boolean", + value: true, + }), + expected: [{ boolean: true }], + }, + { + name: "can find user values", + insert: () => [{ user: config.getUser() }], + query: () => + simpleQuery({ + operator: BasicOperator.EQUAL, + field: "user", + value: config.getUser()._id, }), - searchOpts: { - sort: "number", - sortOrder: SortOrder.ASCENDING, + expected: () => [ + { + user: expect.objectContaining({ _id: config.getUser()._id }), }, - expected: [{ number: 2 }, { number: 3 }], - }, - { - name: "can find numeric values with full range", - insert: [{ number: 1 }, { number: 2 }, { number: 3 }], - query: simpleQuery( - { - operator: "rangeHigh", - field: "number", - value: 2, - }, - { - operator: "rangeLow", - field: "number", - value: 2, - } - ), - expected: [{ number: 2 }], - }, - { - name: "can find longform values", - insert: [{ longform: "foo" }, { longform: "bar" }], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "longform", - value: "foo", - }), - expected: [{ longform: "foo" }], - }, - { - name: "can find options values", - insert: [{ options: "a" }, { options: "b" }], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "options", - value: "a", - }), - expected: [{ options: "a" }], - }, - { - name: "can find array values", - insert: [ - // Number field here is just to guarantee order. - { number: 1, array: ["a"] }, - { number: 2, array: ["b"] }, - { number: 3, array: ["a", "c"] }, - ], - query: simpleQuery({ + ], + }, + { + name: "can find users values", + insert: () => [{ users: [config.getUser()] }], + query: () => + simpleQuery({ operator: ArrayOperator.CONTAINS, - field: "array", - value: "a", + field: "users", + value: [config.getUser()._id], }), - searchOpts: { - sort: "number", - sortOrder: SortOrder.ASCENDING, + expected: () => [ + { + users: [expect.objectContaining({ _id: config.getUser()._id })], }, - expected: [{ array: ["a"] }, { array: ["a", "c"] }], - }, - { - name: "can find bigint values", - insert: [{ bigint: "1" }, { bigint: "2" }], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "bigint", - type: FieldType.BIGINT, - value: "1", - }), - expected: [{ bigint: "1" }], - }, - { - name: "can find datetime values", - insert: [ - { datetime: "2021-01-01T00:00:00.000Z" }, - { datetime: "2021-01-02T00:00:00.000Z" }, - ], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "datetime", - type: FieldType.DATETIME, - value: "2021-01-01", - }), - expected: [{ datetime: "2021-01-01T00:00:00.000Z" }], - }, - { - name: "can find boolean values", - insert: [{ boolean: true }, { boolean: false }], - query: simpleQuery({ - operator: BasicOperator.EQUAL, - field: "boolean", - value: true, - }), - expected: [{ boolean: true }], - }, - { - name: "can find user values", - insert: () => [{ user: config.getUser() }], - query: () => - simpleQuery({ - operator: BasicOperator.EQUAL, - field: "user", - value: config.getUser()._id, - }), - expected: () => [ + ], + }, + { + name: "can handle logical operator any", + insert: [{ string: "bar" }, { string: "foo" }], + query: { + groups: [ { - user: expect.objectContaining({ _id: config.getUser()._id }), - }, - ], - }, - { - name: "can find users values", - insert: () => [{ users: [config.getUser()] }], - query: () => - simpleQuery({ - operator: ArrayOperator.CONTAINS, - field: "users", - value: [config.getUser()._id], - }), - expected: () => [ - { - users: [ - expect.objectContaining({ _id: config.getUser()._id }), + logicalOperator: UILogicalOperator.ANY, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }, + { + operator: BasicOperator.EQUAL, + field: "string", + value: "bar", + }, ], }, ], }, - { - name: "can handle logical operator any", - insert: [{ string: "bar" }, { string: "foo" }], - query: { - groups: [ - { - logicalOperator: UILogicalOperator.ANY, - filters: [ - { - operator: BasicOperator.EQUAL, - field: "string", - value: "foo", - }, - { - operator: BasicOperator.EQUAL, - field: "string", - value: "bar", - }, - ], - }, - ], - }, - searchOpts: { - sort: "string", - sortOrder: SortOrder.ASCENDING, - }, - expected: [{ string: "bar" }, { string: "foo" }], + searchOpts: { + sort: "string", + sortOrder: SortOrder.ASCENDING, }, - { - name: "can handle logical operator all", - insert: [ - { string: "bar", number: 1 }, - { string: "foo", number: 2 }, - ], - query: { - groups: [ - { - logicalOperator: UILogicalOperator.ALL, - filters: [ - { - operator: BasicOperator.EQUAL, - field: "string", - value: "foo", - }, - { - operator: BasicOperator.EQUAL, - field: "number", - value: 2, - }, - ], - }, - ], - }, - searchOpts: { - sort: "string", - sortOrder: SortOrder.ASCENDING, - }, - expected: [{ string: "foo", number: 2 }], - }, - { - name: "overrides allOr with logical operators", - insert: [ - { string: "bar", number: 1 }, - { string: "foo", number: 1 }, - ], - query: { - groups: [ - { - logicalOperator: UILogicalOperator.ALL, - filters: [ - { operator: "allOr" }, - { - operator: BasicOperator.EQUAL, - field: "string", - value: "foo", - }, - { - operator: BasicOperator.EQUAL, - field: "number", - value: 1, - }, - ], - }, - ], - }, - searchOpts: { - sort: "string", - sortOrder: SortOrder.ASCENDING, - }, - expected: [{ string: "foo", number: 1 }], - }, - ] - - it.each(testCases)( - "$name", - async ({ query, insert, expected, searchOpts }) => { - // Some values can't be specified outside of a test (e.g. getting - // config.getUser(), it won't be initialised), so we use functions - // in those cases. - if (typeof insert === "function") { - insert = insert() - } - if (typeof expected === "function") { - expected = expected() - } - if (typeof query === "function") { - query = query() - } - - await config.api.row.bulkImport(table._id!, { rows: insert }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - queryUI: query, - schema: { - string: { visible: true }, - longform: { visible: true }, - options: { visible: true }, - array: { visible: true }, - number: { visible: true }, - bigint: { visible: true }, - datetime: { visible: true }, - boolean: { visible: true }, - user: { visible: true }, - users: { visible: true }, + expected: [{ string: "bar" }, { string: "foo" }], + }, + { + name: "can handle logical operator all", + insert: [ + { string: "bar", number: 1 }, + { string: "foo", number: 2 }, + ], + query: { + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }, + { + operator: BasicOperator.EQUAL, + field: "number", + value: 2, + }, + ], }, - }) + ], + }, + searchOpts: { + sort: "string", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ string: "foo", number: 2 }], + }, + { + name: "overrides allOr with logical operators", + insert: [ + { string: "bar", number: 1 }, + { string: "foo", number: 1 }, + ], + query: { + groups: [ + { + logicalOperator: UILogicalOperator.ALL, + filters: [ + { operator: "allOr" }, + { + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }, + { + operator: BasicOperator.EQUAL, + field: "number", + value: 1, + }, + ], + }, + ], + }, + searchOpts: { + sort: "string", + sortOrder: SortOrder.ASCENDING, + }, + expected: [{ string: "foo", number: 1 }], + }, + ] - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - ...searchOpts, - }) - expect(rows).toEqual( - expected.map(r => expect.objectContaining(r)) - ) + it.each(testCases)( + "$name", + async ({ query, insert, expected, searchOpts }) => { + // Some values can't be specified outside of a test (e.g. getting + // config.getUser(), it won't be initialised), so we use functions + // in those cases. + if (typeof insert === "function") { + insert = insert() } - ) - }) + if (typeof expected === "function") { + expected = expected() + } + if (typeof query === "function") { + query = query() + } + + await config.api.row.bulkImport(table._id!, { rows: insert }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + queryUI: query, + schema: { + string: { visible: true }, + longform: { visible: true }, + options: { visible: true }, + array: { visible: true }, + number: { visible: true }, + bigint: { visible: true }, + datetime: { visible: true }, + boolean: { visible: true }, + user: { visible: true }, + users: { visible: true }, + }, + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + ...searchOpts, + }) + expect(rows).toEqual(expected.map(r => expect.objectContaining(r))) + } + ) }) + }) describe("permissions", () => { beforeEach(async () => { diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index fe44b7b901..1ce519b0b0 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -1,10 +1,6 @@ import * as setup from "../../../api/routes/tests/utilities" import { basicTable } from "../../../tests/utilities/structures" -import { - db as dbCore, - features, - SQLITE_DESIGN_DOC_ID, -} from "@budibase/backend-core" +import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { LinkDocument, DocumentType, @@ -70,24 +66,14 @@ function oldLinkDocument(): Omit { } } -async function sqsDisabled(cb: () => Promise) { - await features.testutils.withFeatureFlags("*", { SQS: false }, cb) -} - -async function sqsEnabled(cb: () => Promise) { - await features.testutils.withFeatureFlags("*", { SQS: true }, cb) -} - describe("SQS migration", () => { beforeAll(async () => { - await sqsDisabled(async () => { - await config.init() - const table = await config.api.table.save(basicTable()) - tableId = table._id! - const db = dbCore.getDB(config.appId!) - // old link document - await db.put(oldLinkDocument()) - }) + await config.init() + const table = await config.api.table.save(basicTable()) + tableId = table._id! + const db = dbCore.getDB(config.appId!) + // old link document + await db.put(oldLinkDocument()) }) beforeEach(async () => { @@ -101,43 +87,32 @@ describe("SQS migration", () => { it("test migration runs as expected against an older DB", async () => { const db = dbCore.getDB(config.appId!) - // confirm nothing exists initially - await sqsDisabled(async () => { - let error: any | undefined - try { - await db.get(SQLITE_DESIGN_DOC_ID) - } catch (err: any) { - error = err - } - expect(error).toBeDefined() - expect(error.status).toBe(404) + + // remove sqlite design doc to simulate it comes from an older installation + const doc = await db.get(SQLITE_DESIGN_DOC_ID) + await db.remove({ _id: doc._id, _rev: doc._rev }) + + await processMigrations(config.appId!, MIGRATIONS) + const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) + expect(designDoc.sql.tables).toBeDefined() + const mainTableDef = designDoc.sql.tables[tableId] + expect(mainTableDef).toBeDefined() + expect(mainTableDef.fields[prefix("name")]).toEqual({ + field: "name", + type: SQLiteType.TEXT, + }) + expect(mainTableDef.fields[prefix("description")]).toEqual({ + field: "description", + type: SQLiteType.TEXT, }) - await sqsEnabled(async () => { - await processMigrations(config.appId!, MIGRATIONS) - const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) - expect(designDoc.sql.tables).toBeDefined() - const mainTableDef = designDoc.sql.tables[tableId] - expect(mainTableDef).toBeDefined() - expect(mainTableDef.fields[prefix("name")]).toEqual({ - field: "name", - type: SQLiteType.TEXT, - }) - expect(mainTableDef.fields[prefix("description")]).toEqual({ - field: "description", - type: SQLiteType.TEXT, - }) - - const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() - const linkDoc = await db.get(oldLinkDocID()) - expect(linkDoc.tableId).toEqual( - generateJunctionTableID(tableId1, tableId2) - ) - // should have swapped the documents - expect(linkDoc.doc1.tableId).toEqual(tableId2) - expect(linkDoc.doc1.rowId).toEqual(rowId2) - expect(linkDoc.doc2.tableId).toEqual(tableId1) - expect(linkDoc.doc2.rowId).toEqual(rowId1) - }) + const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() + const linkDoc = await db.get(oldLinkDocID()) + expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2)) + // should have swapped the documents + expect(linkDoc.doc1.tableId).toEqual(tableId2) + expect(linkDoc.doc1.rowId).toEqual(rowId2) + expect(linkDoc.doc2.tableId).toEqual(tableId1) + expect(linkDoc.doc2.rowId).toEqual(rowId1) }) }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 73ac695878..06a4005061 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -14,11 +14,10 @@ import { coreOutputProcessing, processFormulas, } from "../../utilities/rowProcessor" -import { context, features } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { ContextUser, EventType, - FeatureFlag, FieldType, LinkDocumentValue, Row, @@ -251,19 +250,13 @@ export async function squashLinks( source: Table | ViewV2, enriched: T ): Promise { - const allowRelationshipSchemas = await features.flags.isEnabled( - FeatureFlag.ENRICHED_RELATIONSHIPS - ) - let viewSchema: ViewV2Schema = {} if (sdk.views.isView(source)) { if (helpers.views.isCalculationView(source)) { return enriched } - if (allowRelationshipSchemas) { - viewSchema = source.schema || {} - } + viewSchema = source.schema || {} } let table: Table diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 567e7d5cc8..3a582a46ea 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,11 +1,8 @@ import { EmptyFilterOption, - FeatureFlag, LegacyFilter, - LogicalOperator, Row, RowSearchParams, - SearchFilterKey, SearchFilters, SearchResponse, SortOrder, @@ -19,7 +16,6 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { checkFilters, searchInputMapping } from "./search/utils" -import { db, features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" import { enrichSearchContext } from "../../../api/controllers/row/utils" @@ -104,44 +100,14 @@ export async function search( } viewQuery = checkFilters(table, viewQuery) - const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS) - const supportsLogicalOperators = - isExternalTableID(view.tableId) || sqsEnabled - - if (!supportsLogicalOperators) { - // In the unlikely event that a Grouped Filter is in a non-SQS environment - // It needs to be ignored entirely - let queryFilters: LegacyFilter[] = Array.isArray(view.query) - ? view.query - : [] - - const { filters } = dataFilters.splitFiltersArray(queryFilters) - - // Extract existing fields - const existingFields = filters.map(filter => - db.removeKeyNumbering(filter.field) - ) - - // Carry over filters for unused fields - Object.keys(options.query).forEach(key => { - const operator = key as Exclude - Object.keys(options.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { - viewQuery[operator]![field] = options.query[operator]![field] - } - }) - }) - options.query = viewQuery - } else { - const conditions = viewQuery ? [viewQuery] : [] - options.query = { - $and: { - conditions: [...conditions, options.query], - }, - } - if (viewQuery.onEmptyFilter) { - options.query.onEmptyFilter = viewQuery.onEmptyFilter - } + const conditions = viewQuery ? [viewQuery] : [] + options.query = { + $and: { + conditions: [...conditions, options.query], + }, + } + if (viewQuery.onEmptyFilter) { + options.query.onEmptyFilter = viewQuery.onEmptyFilter } } @@ -170,12 +136,9 @@ export async function search( if (isExternalTable) { span?.addTags({ searchType: "external" }) result = await external.search(options, source) - } else if (await features.flags.isEnabled(FeatureFlag.SQS)) { + } else { span?.addTags({ searchType: "sqs" }) result = await internal.sqs.search(options, source) - } else { - span?.addTags({ searchType: "lucene" }) - result = await internal.lucene.search(options, source) } span.addTags({ diff --git a/packages/server/src/sdk/app/rows/search/internal/index.ts b/packages/server/src/sdk/app/rows/search/internal/index.ts index f3db9169f4..58d1bd9c96 100644 --- a/packages/server/src/sdk/app/rows/search/internal/index.ts +++ b/packages/server/src/sdk/app/rows/search/internal/index.ts @@ -1,3 +1,2 @@ export * as sqs from "./sqs" -export * as lucene from "./lucene" export * from "./internal" diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts deleted file mode 100644 index 953fb90c1f..0000000000 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" -import { fullSearch, paginatedSearch } from "../utils" -import { InternalTables } from "../../../../../db/utils" -import { - Row, - RowSearchParams, - SearchResponse, - SortType, - Table, - User, - ViewV2, -} from "@budibase/types" -import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" -import { outputProcessing } from "../../../../../utilities/rowProcessor" -import pick from "lodash/pick" -import sdk from "../../../../" - -export async function search( - options: RowSearchParams, - source: Table | ViewV2 -): Promise> { - let table: Table - if (sdk.views.isView(source)) { - table = await sdk.views.getTable(source.id) - } else { - table = source - } - - const { paginate, query } = options - - const params: RowSearchParams = { - tableId: options.tableId, - viewId: options.viewId, - sort: options.sort, - sortOrder: options.sortOrder, - sortType: options.sortType, - limit: options.limit, - bookmark: options.bookmark, - version: options.version, - disableEscaping: options.disableEscaping, - query: {}, - } - - if (params.sort && !params.sortType) { - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = - sortField.type === "number" ? SortType.NUMBER : SortType.STRING - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (table._id === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) - } - - const visibleFields = - options.fields || - Object.keys(source.schema || {}).filter( - key => source.schema?.[key].visible !== false - ) - const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS] - response.rows = response.rows.map((r: any) => pick(r, allowedFields)) - - response.rows = await outputProcessing(source, response.rows, { - squash: true, - }) - } - - return response -} diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 4d8a6b6d69..cf91033c40 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,7 +10,7 @@ import { import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" + import { DatabaseName, getDatasource, @@ -21,30 +21,20 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures" // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. describe.each([ - ["lucene", undefined], - ["sqs", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("search sdk (%s)", (name, dsProvider) => { - const isSqs = name === "sqs" - const isLucene = name === "lucene" - const isInternal = isLucene || isSqs + const isInternal = name === "internal" const config = new TestConfiguration() - let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let table: Table beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => - config.init() - ) - - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) + await config.init() if (dsProvider) { datasource = await config.createDatasource({ @@ -105,9 +95,6 @@ describe.each([ afterAll(async () => { config.end() - if (envCleanup) { - envCleanup() - } }) it("querying by fields will always return data attribute columns", async () => { @@ -211,36 +198,35 @@ describe.each([ }) }) - !isLucene && - it.each([ - [["id", "name", "age"], 3], - [["name", "age"], 10], - ])( - "cannot query by non search fields (fields: %s)", - async (queryFields, expectedRows) => { - await config.doInContext(config.appId, async () => { - const { rows } = await search({ - tableId: table._id!, - query: { - $or: { - conditions: [ - { - $and: { - conditions: [ - { range: { id: { low: 2, high: 4 } } }, - { range: { id: { low: 3, high: 5 } } }, - ], - }, + it.each([ + [["id", "name", "age"], 3], + [["name", "age"], 10], + ])( + "cannot query by non search fields (fields: %s)", + async (queryFields, expectedRows) => { + await config.doInContext(config.appId, async () => { + const { rows } = await search({ + tableId: table._id!, + query: { + $or: { + conditions: [ + { + $and: { + conditions: [ + { range: { id: { low: 2, high: 4 } } }, + { range: { id: { low: 3, high: 5 } } }, + ], }, - { equal: { id: 7 } }, - ], - }, + }, + { equal: { id: 7 } }, + ], }, - fields: queryFields, - }) - - expect(rows).toHaveLength(expectedRows) + }, + fields: queryFields, }) - } - ) + + expect(rows).toHaveLength(expectedRows) + }) + } + ) }) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a8ad606647..c59298faf1 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,4 +1,4 @@ -import { context, features } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -12,7 +12,6 @@ import { TableResponse, TableSourceType, TableViewsResponse, - FeatureFlag, } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" @@ -49,10 +48,7 @@ export async function processTable(table: Table): Promise { type: "table", sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, - } - const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS) - if (sqsEnabled) { - processed.sql = true + sql: true, } return processed } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 910e9d220f..907dcb1de4 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils" import { cache, context, - features, HTTPError, objectStore, utils, @@ -19,7 +18,6 @@ import { Table, User, ViewV2, - FeatureFlag, } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { @@ -423,45 +421,43 @@ export async function coreOutputProcessing( // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) - if (isExternal || (await features.flags.isEnabled(FeatureFlag.SQS))) { - for (const row of rows) { - for (const key of Object.keys(row)) { - if (row[key] === null) { - delete row[key] - } else if (row[key] && table.schema[key]?.type === FieldType.LINK) { - for (const link of row[key] || []) { - for (const linkKey of Object.keys(link)) { - if (link[linkKey] === null) { - delete link[linkKey] - } + for (const row of rows) { + for (const key of Object.keys(row)) { + if (row[key] === null) { + delete row[key] + } else if (row[key] && table.schema[key]?.type === FieldType.LINK) { + for (const link of row[key] || []) { + for (const linkKey of Object.keys(link)) { + if (link[linkKey] === null) { + delete link[linkKey] } } } } } + } - if (sdk.views.isView(source)) { - // We ensure calculation fields are returned as numbers. During the - // testing of this feature it was discovered that the COUNT operation - // returns a string for MySQL, MariaDB, and Postgres. But given that all - // calculation fields (except ones operating on BIGINTs) should be - // numbers, we blanket make sure of that here. - for (const [name, field] of Object.entries( - helpers.views.calculationFields(source) - )) { - if ("field" in field) { - const targetSchema = table.schema[field.field] - // We don't convert BIGINT fields to floats because we could lose - // precision. - if (targetSchema.type === FieldType.BIGINT) { - continue - } + if (sdk.views.isView(source)) { + // We ensure calculation fields are returned as numbers. During the + // testing of this feature it was discovered that the COUNT operation + // returns a string for MySQL, MariaDB, and Postgres. But given that all + // calculation fields (except ones operating on BIGINTs) should be + // numbers, we blanket make sure of that here. + for (const [name, field] of Object.entries( + helpers.views.calculationFields(source) + )) { + if ("field" in field) { + const targetSchema = table.schema[field.field] + // We don't convert BIGINT fields to floats because we could lose + // precision. + if (targetSchema.type === FieldType.BIGINT) { + continue } + } - for (const row of rows) { - if (typeof row[name] === "string") { - row[name] = parseFloat(row[name]) - } + for (const row of rows) { + if (typeof row[name] === "string") { + row[name] = parseFloat(row[name]) } } } diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 8cbe585d90..cd375ecb23 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -8,7 +8,7 @@ import { } from "@budibase/types" import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" + import * as bbReferenceProcessor from "../bbReferenceProcessor" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -21,7 +21,6 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ describe("rowProcessor - outputProcessing", () => { const config = new TestConfiguration() - let cleanupFlags: () => void = () => {} beforeAll(async () => { await config.init() @@ -33,11 +32,6 @@ describe("rowProcessor - outputProcessing", () => { beforeEach(() => { jest.resetAllMocks() - cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true }) - }) - - afterEach(() => { - cleanupFlags() }) const processOutputBBReferenceMock = diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 15c30800a1..61950fd523 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -527,7 +527,12 @@ export function search>( ): SearchResponse { let result = runQuery(docs, query.query) if (query.sort) { - result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) + result = sort( + result, + query.sort, + query.sortOrder || SortOrder.ASCENDING, + query.sortType + ) } const totalRows = result.length if (query.limit) { diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index b679d6e182..9797715329 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -12,7 +12,6 @@ import type PouchDB from "pouchdb-find" export enum SearchIndex { ROWS = "rows", - AUDIT = "audit", USER = "user", } diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 97d145db6c..64a7362e9f 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -2,10 +2,9 @@ export enum FeatureFlag { PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE", PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT", AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING", - SQS = "SQS", AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS", DEFAULT_VALUES = "DEFAULT_VALUES", - ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS", + BUDIBASE_AI = "BUDIBASE_AI", } diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index 48ab2b586f..18ecc380db 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,6 @@ -import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types" +import { Ctx, MaintenanceType } from "@budibase/types" import env from "../../../environment" -import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core" +import { env as coreEnv, db as dbCore } from "@budibase/backend-core" import nodeFetch from "node-fetch" import { helpers } from "@budibase/shared-core" @@ -35,10 +35,7 @@ async function isSqsAvailable() { } async function isSqsMissing() { - return ( - (await features.flags.isEnabled(FeatureFlag.SQS)) && - !(await isSqsAvailable()) - ) + return !(await isSqsAvailable()) } export const fetch = async (ctx: Ctx) => { diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index b540836583..f901925016 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,5 +1,5 @@ import { mocks, structures } from "@budibase/backend-core/tests" -import { context, events, features } from "@budibase/backend-core" +import { context, events } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -12,19 +12,14 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { +describe("/api/global/auditlogs (%s)", () => { const config = new TestConfiguration() - let envCleanup: (() => void) | undefined beforeAll(async () => { - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: method === "sql", - }) await config.beforeAll() }) afterAll(async () => { - envCleanup?.() await config.afterAll() })