Merge remote-tracking branch 'origin/master' into automation-branching-ux-updates

hosting/nginx.dev.conf

@@ -45,20 +45,6 @@ http {
     client_max_body_size 50000m;
     ignore_invalid_headers off;
     proxy_buffering off;
-    set $csp_default "default-src 'self'";
-    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
-    set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
-    set $csp_object "object-src 'none'";
-    set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
-    set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
-    set $csp_frame "frame-src 'self' https:";
-    set $csp_img "img-src http: https: data: blob:";
-    set $csp_manifest "manifest-src 'self'";
-    set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live";
-    set $csp_worker "worker-src blob:";
-
-    add_header Content-Security-Policy "${csp_default}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;
 
     error_page 502 503 504 /error.html;
     location = /error.html {

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -10,7 +10,6 @@ import {
   DatabaseQueryOpts,
   DBError,
   Document,
-  FeatureFlag,
   isDocument,
   RowResponse,
   RowValue,
@@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 import { checkSlashesInUrl } from "../../helpers"
 import { sqlLog } from "../../sql/utils"
-import { flags } from "../../features"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
 
@@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
-    if (
+    if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
-      (await flags.isEnabled(FeatureFlag.SQS)) &&
-      (await this.exists(SQLITE_DESIGN_DOC_ID))
-    ) {
       // delete the design document, then run the cleanup operation
       const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
       // remove all tables - save the definition then trigger a cleanup

packages/backend-core/src/features/features.ts

@@ -269,8 +269,6 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
   [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
-  [FeatureFlag.SQS]: Flag.boolean(true),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
   [FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),
 })

packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte

@@ -11,7 +11,6 @@
   export let disabledPermissions = []
   export let columns
   export let fromRelationshipField
-  export let canSetRelationshipSchemas
 
   const { datasource, dispatch } = getContext("grid")
 
@@ -129,6 +128,8 @@
     }
   })
 
+  $: hasLinkColumns = columns.some(c => c.schema.type === FieldType.LINK)
+
   async function toggleColumn(column, permission) {
     const visible = permission !== FieldPermissions.HIDDEN
     const readonly = permission === FieldPermissions.READONLY
@@ -184,7 +185,7 @@
           value={columnToPermissionOptions(column)}
           options={column.options}
         />
-        {#if canSetRelationshipSchemas && column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
+        {#if column.schema.type === FieldType.LINK && columnToPermissionOptions(column) !== FieldPermissions.HIDDEN}
           <div class="relationship-columns">
             <ActionButton
               on:click={e => {
@@ -203,7 +204,7 @@
   </div>
 </div>
 
-{#if canSetRelationshipSchemas}
+{#if hasLinkColumns}
   <Popover
     on:close={() => (relationshipFieldName = null)}
     open={relationshipFieldName}

packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte

@@ -10,8 +10,6 @@
   import { getContext } from "svelte"
   import { ActionButton } from "@budibase/bbui"
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
-  import { isEnabled } from "helpers/featureFlags"
-  import { FeatureFlag } from "@budibase/types"
   import DetailPopover from "components/common/DetailPopover.svelte"
 
   const { tableColumns, datasource } = getContext("grid")
@@ -46,9 +44,5 @@
       {text}
     </ActionButton>
   </svelte:fragment>
-  <ColumnsSettingContent
+  <ColumnsSettingContent columns={$tableColumns} {permissions} />
-    columns={$tableColumns}
-    canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
-    {permissions}
-  />
 </DetailPopover>

packages/pro

@@ -1 +1 @@
-Subproject commit 80770215c6159e4d47f3529fd02e74bc8ad07543
+Subproject commit a56696a4af5667617746600fc75fe6a01744b692

packages/server/src/api/controllers/table/utils.ts

@@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { context, events, features, HTTPError } from "@budibase/backend-core"
+import { context, events, HTTPError } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   Database,
   Datasource,
-  FeatureFlag,
   FieldSchema,
   FieldType,
   NumberFieldMetadata,
@@ -336,9 +335,8 @@ class TableSaveFunctions {
       importRows: this.importRows,
       userId: this.userId,
     })
-    if (await features.flags.isEnabled(FeatureFlag.SQS)) {
+
-      await sdk.tables.sqs.addTable(table)
+    await sdk.tables.sqs.addTable(table)
-    }
     return table
   }
 
@@ -530,9 +528,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
   if (rows) {
     await AttachmentCleanup.tableDelete(table, rows)
   }
-  if (await features.flags.isEnabled(FeatureFlag.SQS)) {
+
-    await sdk.tables.sqs.removeTable(table)
+  await sdk.tables.sqs.removeTable(table)
-  }
 }
 
 const _TableSaveFunctions = TableSaveFunctions

packages/server/src/api/routes/tests/application.spec.ts

@@ -16,7 +16,7 @@ jest.mock("../../../utilities/redis", () => ({
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
-import { events, utils, context, features } from "@budibase/backend-core"
+import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
 import { type App, BuiltinPermissionID } from "@budibase/types"
 import tk from "timekeeper"
@@ -355,21 +355,6 @@ describe("/applications", () => {
       expect(events.app.deleted).toHaveBeenCalledTimes(1)
       expect(events.app.unpublished).toHaveBeenCalledTimes(1)
     })
-
-    it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => {
-      const prodAppId = app.appId.replace("_dev", "")
-      nock("http://localhost:10000")
-        .delete(`/api/global/roles/${prodAppId}`)
-        .reply(200, {})
-
-      await features.testutils.withFeatureFlags(
-        "*",
-        { SQS: true },
-        async () => {
-          await config.api.application.delete(app.appId)
-        }
-      )
-    })
   })
 
   describe("POST /api/applications/:appId/duplicate", () => {

packages/server/src/api/routes/tests/row.spec.ts

@@ -9,27 +9,18 @@ import {
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
 import { outputProcessing } from "../../../utilities/rowProcessor"
-import {
+import { context, InternalTable, tenancy, utils } from "@budibase/backend-core"
-  context,
-  InternalTable,
-  tenancy,
-  features,
-  utils,
-} from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   AIOperationEnum,
-  AttachmentFieldMetadata,
   AutoFieldSubType,
   Datasource,
-  DateFieldMetadata,
   DeleteRow,
   FieldSchema,
   FieldType,
   BBReferenceFieldSubType,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
-  NumberFieldMetadata,
   QuotaUsageType,
   RelationshipType,
   Row,
@@ -90,8 +81,7 @@ async function waitForEvent(
 }
 
 describe.each([
-  ["lucene", undefined],
+  ["internal", undefined],
-  ["sqs", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
@@ -99,8 +89,6 @@ describe.each([
   [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
-  const isLucene = providerType === "lucene"
-  const isSqs = providerType === "sqs"
   const isMSSQL = providerType === DatabaseName.SQL_SERVER
   const isOracle = providerType === DatabaseName.ORACLE
   const config = setup.getConfig()
@@ -108,15 +96,9 @@ describe.each([
   let table: Table
   let datasource: Datasource | undefined
   let client: Knex | undefined
-  let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
+    await config.init()
-      config.init()
-    )
-    envCleanup = features.testutils.setFeatureFlags("*", {
-      SQS: isSqs,
-    })
 
     if (dsProvider) {
       const rawDatasource = await dsProvider
@@ -129,9 +111,6 @@ describe.each([
 
   afterAll(async () => {
     setup.afterAll()
-    if (envCleanup) {
-      envCleanup()
-    }
   })
 
   function saveTableRequest(
@@ -381,185 +360,6 @@ describe.each([
         expect(ids).toEqual(expect.arrayContaining(sequence))
       })
 
-    isLucene &&
-      it("row values are coerced", async () => {
-        const str: FieldSchema = {
-          type: FieldType.STRING,
-          name: "str",
-          constraints: { type: "string", presence: false },
-        }
-        const singleAttachment: FieldSchema = {
-          type: FieldType.ATTACHMENT_SINGLE,
-          name: "single attachment",
-          constraints: { presence: false },
-        }
-        const attachmentList: AttachmentFieldMetadata = {
-          type: FieldType.ATTACHMENTS,
-          name: "attachments",
-          constraints: { type: "array", presence: false },
-        }
-        const signature: FieldSchema = {
-          type: FieldType.SIGNATURE_SINGLE,
-          name: "signature",
-          constraints: { presence: false },
-        }
-        const bool: FieldSchema = {
-          type: FieldType.BOOLEAN,
-          name: "boolean",
-          constraints: { type: "boolean", presence: false },
-        }
-        const number: NumberFieldMetadata = {
-          type: FieldType.NUMBER,
-          name: "str",
-          constraints: { type: "number", presence: false },
-        }
-        const datetime: DateFieldMetadata = {
-          type: FieldType.DATETIME,
-          name: "datetime",
-          constraints: {
-            type: "string",
-            presence: false,
-            datetime: { earliest: "", latest: "" },
-          },
-        }
-        const arrayField: FieldSchema = {
-          type: FieldType.ARRAY,
-          constraints: {
-            type: JsonFieldSubType.ARRAY,
-            presence: false,
-            inclusion: ["One", "Two", "Three"],
-          },
-          name: "Sample Tags",
-          sortable: false,
-        }
-        const optsField: FieldSchema = {
-          name: "Sample Opts",
-          type: FieldType.OPTIONS,
-          constraints: {
-            type: "string",
-            presence: false,
-            inclusion: ["Alpha", "Beta", "Gamma"],
-          },
-        }
-        const table = await config.api.table.save(
-          saveTableRequest({
-            schema: {
-              name: str,
-              stringUndefined: str,
-              stringNull: str,
-              stringString: str,
-              numberEmptyString: number,
-              numberNull: number,
-              numberUndefined: number,
-              numberString: number,
-              numberNumber: number,
-              datetimeEmptyString: datetime,
-              datetimeNull: datetime,
-              datetimeUndefined: datetime,
-              datetimeString: datetime,
-              datetimeDate: datetime,
-              boolNull: bool,
-              boolEmpty: bool,
-              boolUndefined: bool,
-              boolString: bool,
-              boolBool: bool,
-              singleAttachmentNull: singleAttachment,
-              singleAttachmentUndefined: singleAttachment,
-              attachmentListNull: attachmentList,
-              attachmentListUndefined: attachmentList,
-              attachmentListEmpty: attachmentList,
-              attachmentListEmptyArrayStr: attachmentList,
-              signatureNull: signature,
-              signatureUndefined: signature,
-              arrayFieldEmptyArrayStr: arrayField,
-              arrayFieldArrayStrKnown: arrayField,
-              arrayFieldNull: arrayField,
-              arrayFieldUndefined: arrayField,
-              optsFieldEmptyStr: optsField,
-              optsFieldUndefined: optsField,
-              optsFieldNull: optsField,
-              optsFieldStrKnown: optsField,
-            },
-          })
-        )
-
-        const datetimeStr = "1984-04-20T00:00:00.000Z"
-
-        const row = await config.api.row.save(table._id!, {
-          name: "Test Row",
-          stringUndefined: undefined,
-          stringNull: null,
-          stringString: "i am a string",
-          numberEmptyString: "",
-          numberNull: null,
-          numberUndefined: undefined,
-          numberString: "123",
-          numberNumber: 123,
-          datetimeEmptyString: "",
-          datetimeNull: null,
-          datetimeUndefined: undefined,
-          datetimeString: datetimeStr,
-          datetimeDate: new Date(datetimeStr),
-          boolNull: null,
-          boolEmpty: "",
-          boolUndefined: undefined,
-          boolString: "true",
-          boolBool: true,
-          tableId: table._id,
-          singleAttachmentNull: null,
-          singleAttachmentUndefined: undefined,
-          attachmentListNull: null,
-          attachmentListUndefined: undefined,
-          attachmentListEmpty: "",
-          attachmentListEmptyArrayStr: "[]",
-          signatureNull: null,
-          signatureUndefined: undefined,
-          arrayFieldEmptyArrayStr: "[]",
-          arrayFieldUndefined: undefined,
-          arrayFieldNull: null,
-          arrayFieldArrayStrKnown: "['One']",
-          optsFieldEmptyStr: "",
-          optsFieldUndefined: undefined,
-          optsFieldNull: null,
-          optsFieldStrKnown: "Alpha",
-        })
-
-        expect(row.stringUndefined).toBe(undefined)
-        expect(row.stringNull).toBe(null)
-        expect(row.stringString).toBe("i am a string")
-        expect(row.numberEmptyString).toBe(null)
-        expect(row.numberNull).toBe(null)
-        expect(row.numberUndefined).toBe(undefined)
-        expect(row.numberString).toBe(123)
-        expect(row.numberNumber).toBe(123)
-        expect(row.datetimeEmptyString).toBe(null)
-        expect(row.datetimeNull).toBe(null)
-        expect(row.datetimeUndefined).toBe(undefined)
-        expect(row.datetimeString).toBe(new Date(datetimeStr).toISOString())
-        expect(row.datetimeDate).toBe(new Date(datetimeStr).toISOString())
-        expect(row.boolNull).toBe(null)
-        expect(row.boolEmpty).toBe(null)
-        expect(row.boolUndefined).toBe(undefined)
-        expect(row.boolString).toBe(true)
-        expect(row.boolBool).toBe(true)
-        expect(row.singleAttachmentNull).toEqual(null)
-        expect(row.singleAttachmentUndefined).toBe(undefined)
-        expect(row.attachmentListNull).toEqual([])
-        expect(row.attachmentListUndefined).toBe(undefined)
-        expect(row.attachmentListEmpty).toEqual([])
-        expect(row.attachmentListEmptyArrayStr).toEqual([])
-        expect(row.signatureNull).toEqual(null)
-        expect(row.signatureUndefined).toBe(undefined)
-        expect(row.arrayFieldEmptyArrayStr).toEqual([])
-        expect(row.arrayFieldNull).toEqual([])
-        expect(row.arrayFieldUndefined).toEqual(undefined)
-        expect(row.optsFieldEmptyStr).toEqual(null)
-        expect(row.optsFieldUndefined).toEqual(undefined)
-        expect(row.optsFieldNull).toEqual(null)
-        expect(row.arrayFieldArrayStrKnown).toEqual(["One"])
-        expect(row.optsFieldStrKnown).toEqual("Alpha")
-      })
-
     isInternal &&
       it("doesn't allow creating in user table", async () => {
         const response = await config.api.row.save(
@@ -1023,104 +823,103 @@ describe.each([
       })
     })
 
-    !isLucene &&
+    describe("relations to same table", () => {
-      describe("relations to same table", () => {
+      let relatedRows: Row[]
-        let relatedRows: Row[]
 
-        beforeAll(async () => {
+      beforeAll(async () => {
-          const relatedTable = await config.api.table.save(
+        const relatedTable = await config.api.table.save(
-            defaultTable({
+          defaultTable({
-              schema: {
+            schema: {
-                name: { name: "name", type: FieldType.STRING },
+              name: { name: "name", type: FieldType.STRING },
+            },
+          })
+        )
+        const relatedTableId = relatedTable._id!
+        table = await config.api.table.save(
+          defaultTable({
+            schema: {
+              name: { name: "name", type: FieldType.STRING },
+              related1: {
+                type: FieldType.LINK,
+                name: "related1",
+                fieldName: "main1",
+                tableId: relatedTableId,
+                relationshipType: RelationshipType.MANY_TO_MANY,
               },
-            })
+              related2: {
-          )
+                type: FieldType.LINK,
-          const relatedTableId = relatedTable._id!
+                name: "related2",
-          table = await config.api.table.save(
+                fieldName: "main2",
-            defaultTable({
+                tableId: relatedTableId,
-              schema: {
+                relationshipType: RelationshipType.MANY_TO_MANY,
-                name: { name: "name", type: FieldType.STRING },
-                related1: {
-                  type: FieldType.LINK,
-                  name: "related1",
-                  fieldName: "main1",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
-                related2: {
-                  type: FieldType.LINK,
-                  name: "related2",
-                  fieldName: "main2",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
               },
-            })
+            },
-          )
-          relatedRows = await Promise.all([
-            config.api.row.save(relatedTableId, { name: "foo" }),
-            config.api.row.save(relatedTableId, { name: "bar" }),
-            config.api.row.save(relatedTableId, { name: "baz" }),
-            config.api.row.save(relatedTableId, { name: "boo" }),
-          ])
-        })
-
-        it("can create rows with both relationships", async () => {
-          const row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [relatedRows[0]._id!],
-            related2: [relatedRows[1]._id!],
           })
-
+        )
-          expect(row).toEqual(
+        relatedRows = await Promise.all([
-            expect.objectContaining({
+          config.api.row.save(relatedTableId, { name: "foo" }),
-              name: "test",
+          config.api.row.save(relatedTableId, { name: "bar" }),
-              related1: [
+          config.api.row.save(relatedTableId, { name: "baz" }),
-                {
+          config.api.row.save(relatedTableId, { name: "boo" }),
-                  _id: relatedRows[0]._id,
+        ])
-                  primaryDisplay: relatedRows[0].name,
-                },
-              ],
-              related2: [
-                {
-                  _id: relatedRows[1]._id,
-                  primaryDisplay: relatedRows[1].name,
-                },
-              ],
-            })
-          )
-        })
-
-        it("can create rows with no relationships", async () => {
-          const row = await config.api.row.save(table._id!, {
-            name: "test",
-          })
-
-          expect(row.related1).toBeUndefined()
-          expect(row.related2).toBeUndefined()
-        })
-
-        it("can create rows with only one relationships field", async () => {
-          const row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [],
-            related2: [relatedRows[1]._id!],
-          })
-
-          expect(row).toEqual(
-            expect.objectContaining({
-              name: "test",
-              related2: [
-                {
-                  _id: relatedRows[1]._id,
-                  primaryDisplay: relatedRows[1].name,
-                },
-              ],
-            })
-          )
-          expect(row.related1).toBeUndefined()
-        })
       })
+
+      it("can create rows with both relationships", async () => {
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [relatedRows[0]._id!],
+          related2: [relatedRows[1]._id!],
+        })
+
+        expect(row).toEqual(
+          expect.objectContaining({
+            name: "test",
+            related1: [
+              {
+                _id: relatedRows[0]._id,
+                primaryDisplay: relatedRows[0].name,
+              },
+            ],
+            related2: [
+              {
+                _id: relatedRows[1]._id,
+                primaryDisplay: relatedRows[1].name,
+              },
+            ],
+          })
+        )
+      })
+
+      it("can create rows with no relationships", async () => {
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+        })
+
+        expect(row.related1).toBeUndefined()
+        expect(row.related2).toBeUndefined()
+      })
+
+      it("can create rows with only one relationships field", async () => {
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [],
+          related2: [relatedRows[1]._id!],
+        })
+
+        expect(row).toEqual(
+          expect.objectContaining({
+            name: "test",
+            related2: [
+              {
+                _id: relatedRows[1]._id,
+                primaryDisplay: relatedRows[1].name,
+              },
+            ],
+          })
+        )
+        expect(row.related1).toBeUndefined()
+      })
+    })
   })
 
   describe("get", () => {
@@ -1224,133 +1023,132 @@ describe.each([
         expect(rows).toHaveLength(1)
       })
 
-    !isLucene &&
+    describe("relations to same table", () => {
-      describe("relations to same table", () => {
+      let relatedRows: Row[]
-        let relatedRows: Row[]
 
-        beforeAll(async () => {
+      beforeAll(async () => {
-          const relatedTable = await config.api.table.save(
+        const relatedTable = await config.api.table.save(
-            defaultTable({
+          defaultTable({
-              schema: {
+            schema: {
-                name: { name: "name", type: FieldType.STRING },
+              name: { name: "name", type: FieldType.STRING },
+            },
+          })
+        )
+        const relatedTableId = relatedTable._id!
+        table = await config.api.table.save(
+          defaultTable({
+            schema: {
+              name: { name: "name", type: FieldType.STRING },
+              related1: {
+                type: FieldType.LINK,
+                name: "related1",
+                fieldName: "main1",
+                tableId: relatedTableId,
+                relationshipType: RelationshipType.MANY_TO_MANY,
               },
-            })
+              related2: {
-          )
+                type: FieldType.LINK,
-          const relatedTableId = relatedTable._id!
+                name: "related2",
-          table = await config.api.table.save(
+                fieldName: "main2",
-            defaultTable({
+                tableId: relatedTableId,
-              schema: {
+                relationshipType: RelationshipType.MANY_TO_MANY,
-                name: { name: "name", type: FieldType.STRING },
-                related1: {
-                  type: FieldType.LINK,
-                  name: "related1",
-                  fieldName: "main1",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
-                related2: {
-                  type: FieldType.LINK,
-                  name: "related2",
-                  fieldName: "main2",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
               },
-            })
+            },
-          )
-          relatedRows = await Promise.all([
-            config.api.row.save(relatedTableId, { name: "foo" }),
-            config.api.row.save(relatedTableId, { name: "bar" }),
-            config.api.row.save(relatedTableId, { name: "baz" }),
-            config.api.row.save(relatedTableId, { name: "boo" }),
-          ])
-        })
-
-        it("can edit rows with both relationships", async () => {
-          let row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [relatedRows[0]._id!],
-            related2: [relatedRows[1]._id!],
           })
-
+        )
-          row = await config.api.row.save(table._id!, {
+        relatedRows = await Promise.all([
-            ...row,
+          config.api.row.save(relatedTableId, { name: "foo" }),
-            related1: [relatedRows[0]._id!, relatedRows[1]._id!],
+          config.api.row.save(relatedTableId, { name: "bar" }),
-            related2: [relatedRows[2]._id!],
+          config.api.row.save(relatedTableId, { name: "baz" }),
-          })
+          config.api.row.save(relatedTableId, { name: "boo" }),
-
+        ])
-          expect(row).toEqual(
-            expect.objectContaining({
-              name: "test",
-              related1: expect.arrayContaining([
-                {
-                  _id: relatedRows[0]._id,
-                  primaryDisplay: relatedRows[0].name,
-                },
-                {
-                  _id: relatedRows[1]._id,
-                  primaryDisplay: relatedRows[1].name,
-                },
-              ]),
-              related2: [
-                {
-                  _id: relatedRows[2]._id,
-                  primaryDisplay: relatedRows[2].name,
-                },
-              ],
-            })
-          )
-        })
-
-        it("can drop existing relationship", async () => {
-          let row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [relatedRows[0]._id!],
-            related2: [relatedRows[1]._id!],
-          })
-
-          row = await config.api.row.save(table._id!, {
-            ...row,
-            related1: [],
-            related2: [relatedRows[2]._id!],
-          })
-
-          expect(row).toEqual(
-            expect.objectContaining({
-              name: "test",
-              related2: [
-                {
-                  _id: relatedRows[2]._id,
-                  primaryDisplay: relatedRows[2].name,
-                },
-              ],
-            })
-          )
-          expect(row.related1).toBeUndefined()
-        })
-
-        it("can drop both relationships", async () => {
-          let row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [relatedRows[0]._id!],
-            related2: [relatedRows[1]._id!],
-          })
-
-          row = await config.api.row.save(table._id!, {
-            ...row,
-            related1: [],
-            related2: [],
-          })
-
-          expect(row).toEqual(
-            expect.objectContaining({
-              name: "test",
-            })
-          )
-          expect(row.related1).toBeUndefined()
-          expect(row.related2).toBeUndefined()
-        })
       })
+
+      it("can edit rows with both relationships", async () => {
+        let row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [relatedRows[0]._id!],
+          related2: [relatedRows[1]._id!],
+        })
+
+        row = await config.api.row.save(table._id!, {
+          ...row,
+          related1: [relatedRows[0]._id!, relatedRows[1]._id!],
+          related2: [relatedRows[2]._id!],
+        })
+
+        expect(row).toEqual(
+          expect.objectContaining({
+            name: "test",
+            related1: expect.arrayContaining([
+              {
+                _id: relatedRows[0]._id,
+                primaryDisplay: relatedRows[0].name,
+              },
+              {
+                _id: relatedRows[1]._id,
+                primaryDisplay: relatedRows[1].name,
+              },
+            ]),
+            related2: [
+              {
+                _id: relatedRows[2]._id,
+                primaryDisplay: relatedRows[2].name,
+              },
+            ],
+          })
+        )
+      })
+
+      it("can drop existing relationship", async () => {
+        let row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [relatedRows[0]._id!],
+          related2: [relatedRows[1]._id!],
+        })
+
+        row = await config.api.row.save(table._id!, {
+          ...row,
+          related1: [],
+          related2: [relatedRows[2]._id!],
+        })
+
+        expect(row).toEqual(
+          expect.objectContaining({
+            name: "test",
+            related2: [
+              {
+                _id: relatedRows[2]._id,
+                primaryDisplay: relatedRows[2].name,
+              },
+            ],
+          })
+        )
+        expect(row.related1).toBeUndefined()
+      })
+
+      it("can drop both relationships", async () => {
+        let row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [relatedRows[0]._id!],
+          related2: [relatedRows[1]._id!],
+        })
+
+        row = await config.api.row.save(table._id!, {
+          ...row,
+          related1: [],
+          related2: [],
+        })
+
+        expect(row).toEqual(
+          expect.objectContaining({
+            name: "test",
+          })
+        )
+        expect(row.related1).toBeUndefined()
+        expect(row.related2).toBeUndefined()
+      })
+    })
   })
 
   describe("patch", () => {
@@ -1628,72 +1426,71 @@ describe.each([
       expect(res.length).toEqual(2)
     })
 
-    !isLucene &&
+    describe("relations to same table", () => {
-      describe("relations to same table", () => {
+      let relatedRows: Row[]
-        let relatedRows: Row[]
 
-        beforeAll(async () => {
+      beforeAll(async () => {
-          const relatedTable = await config.api.table.save(
+        const relatedTable = await config.api.table.save(
-            defaultTable({
+          defaultTable({
-              schema: {
+            schema: {
-                name: { name: "name", type: FieldType.STRING },
+              name: { name: "name", type: FieldType.STRING },
-              },
+            },
-            })
-          )
-          const relatedTableId = relatedTable._id!
-          table = await config.api.table.save(
-            defaultTable({
-              schema: {
-                name: { name: "name", type: FieldType.STRING },
-                related1: {
-                  type: FieldType.LINK,
-                  name: "related1",
-                  fieldName: "main1",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
-                related2: {
-                  type: FieldType.LINK,
-                  name: "related2",
-                  fieldName: "main2",
-                  tableId: relatedTableId,
-                  relationshipType: RelationshipType.MANY_TO_MANY,
-                },
-              },
-            })
-          )
-          relatedRows = await Promise.all([
-            config.api.row.save(relatedTableId, { name: "foo" }),
-            config.api.row.save(relatedTableId, { name: "bar" }),
-            config.api.row.save(relatedTableId, { name: "baz" }),
-            config.api.row.save(relatedTableId, { name: "boo" }),
-          ])
-        })
-
-        it("can delete rows with both relationships", async () => {
-          const row = await config.api.row.save(table._id!, {
-            name: "test",
-            related1: [relatedRows[0]._id!],
-            related2: [relatedRows[1]._id!],
           })
-
+        )
-          await config.api.row.delete(table._id!, { _id: row._id! })
+        const relatedTableId = relatedTable._id!
-
+        table = await config.api.table.save(
-          await config.api.row.get(table._id!, row._id!, { status: 404 })
+          defaultTable({
-        })
+            schema: {
-
+              name: { name: "name", type: FieldType.STRING },
-        it("can delete rows with empty relationships", async () => {
+              related1: {
-          const row = await config.api.row.save(table._id!, {
+                type: FieldType.LINK,
-            name: "test",
+                name: "related1",
-            related1: [],
+                fieldName: "main1",
-            related2: [],
+                tableId: relatedTableId,
+                relationshipType: RelationshipType.MANY_TO_MANY,
+              },
+              related2: {
+                type: FieldType.LINK,
+                name: "related2",
+                fieldName: "main2",
+                tableId: relatedTableId,
+                relationshipType: RelationshipType.MANY_TO_MANY,
+              },
+            },
           })
-
+        )
-          await config.api.row.delete(table._id!, { _id: row._id! })
+        relatedRows = await Promise.all([
-
+          config.api.row.save(relatedTableId, { name: "foo" }),
-          await config.api.row.get(table._id!, row._id!, { status: 404 })
+          config.api.row.save(relatedTableId, { name: "bar" }),
-        })
+          config.api.row.save(relatedTableId, { name: "baz" }),
+          config.api.row.save(relatedTableId, { name: "boo" }),
+        ])
       })
+
+      it("can delete rows with both relationships", async () => {
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [relatedRows[0]._id!],
+          related2: [relatedRows[1]._id!],
+        })
+
+        await config.api.row.delete(table._id!, { _id: row._id! })
+
+        await config.api.row.get(table._id!, row._id!, { status: 404 })
+      })
+
+      it("can delete rows with empty relationships", async () => {
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          related1: [],
+          related2: [],
+        })
+
+        await config.api.row.delete(table._id!, { _id: row._id! })
+
+        await config.api.row.get(table._id!, row._id!, { status: 404 })
+      })
+    })
   })
 
   describe("validate", () => {
@@ -3061,13 +2858,7 @@ describe.each([
 
       let auxData: Row[] = []
 
-      let flagCleanup: (() => void) | undefined
-
       beforeAll(async () => {
-        flagCleanup = features.testutils.setFeatureFlags("*", {
-          ENRICHED_RELATIONSHIPS: true,
-        })
-
         const aux2Table = await config.api.table.save(saveTableRequest())
         const aux2Data = await config.api.row.save(aux2Table._id!, {})
 
@@ -3214,10 +3005,6 @@ describe.each([
         viewId = view.id
       })
 
-      afterAll(() => {
-        flagCleanup?.()
-      })
-
       const testScenarios: [string, (row: Row) => Promise<Row> | Row][] = [
         ["get row", (row: Row) => config.api.row.get(viewId, row._id!)],
         [
@@ -3290,68 +3077,6 @@ describe.each([
         }
       )
 
-      it.each(testScenarios)(
-        "does not enrich relationships when not enabled (via %s)",
-        async (__, retrieveDelegate) => {
-          await features.testutils.withFeatureFlags(
-            "*",
-            {
-              ENRICHED_RELATIONSHIPS: false,
-            },
-            async () => {
-              const otherRows = _.sampleSize(auxData, 5)
-
-              const row = await config.api.row.save(viewId, {
-                title: generator.word(),
-                relWithNoSchema: [otherRows[0]],
-                relWithEmptySchema: [otherRows[1]],
-                relWithFullSchema: [otherRows[2]],
-                relWithHalfSchema: [otherRows[3]],
-                relWithIllegalSchema: [otherRows[4]],
-              })
-
-              const retrieved = await retrieveDelegate(row)
-
-              expect(retrieved).toEqual(
-                expect.objectContaining({
-                  title: row.title,
-                  relWithNoSchema: [
-                    {
-                      _id: otherRows[0]._id,
-                      primaryDisplay: otherRows[0].name,
-                    },
-                  ],
-                  relWithEmptySchema: [
-                    {
-                      _id: otherRows[1]._id,
-                      primaryDisplay: otherRows[1].name,
-                    },
-                  ],
-                  relWithFullSchema: [
-                    {
-                      _id: otherRows[2]._id,
-                      primaryDisplay: otherRows[2].name,
-                    },
-                  ],
-                  relWithHalfSchema: [
-                    {
-                      _id: otherRows[3]._id,
-                      primaryDisplay: otherRows[3].name,
-                    },
-                  ],
-                  relWithIllegalSchema: [
-                    {
-                      _id: otherRows[4]._id,
-                      primaryDisplay: otherRows[4].name,
-                    },
-                  ],
-                })
-              )
-            }
-          )
-        }
-      )
-
       it.each([
         [
           "from table fetch",
@@ -3422,7 +3147,7 @@ describe.each([
       )
     })
 
-  isSqs &&
+  isInternal &&
     describe("AI fields", () => {
       let table: Table
 

packages/server/src/api/routes/tests/templates.spec.ts

@@ -2,7 +2,6 @@ import * as setup from "./utilities"
 import path from "path"
 import nock from "nock"
 import { generator } from "@budibase/backend-core/tests"
-import { features } from "@budibase/backend-core"
 
 interface App {
   background: string
@@ -82,48 +81,36 @@ describe("/templates", () => {
   })
 
   describe("create app from template", () => {
-    it.each(["sqs", "lucene"])(
+    it("should be able to create an app from a template", async () => {
-      `should be able to create an app from a template (%s)`,
+      const name = generator.guid().replaceAll("-", "")
-      async source => {
+      const url = `/${name}`
-        await features.testutils.withFeatureFlags(
-          "*",
-          { SQS: source === "sqs" },
-          async () => {
-            const name = generator.guid().replaceAll("-", "")
-            const url = `/${name}`
 
-            const app = await config.api.application.create({
+      const app = await config.api.application.create({
-              name,
+        name,
-              url,
+        url,
-              useTemplate: "true",
+        useTemplate: "true",
-              templateName: "Agency Client Portal",
+        templateName: "Agency Client Portal",
-              templateKey: "app/agency-client-portal",
+        templateKey: "app/agency-client-portal",
-            })
+      })
-            expect(app.name).toBe(name)
+      expect(app.name).toBe(name)
-            expect(app.url).toBe(url)
+      expect(app.url).toBe(url)
 
-            await config.withApp(app, async () => {
+      await config.withApp(app, async () => {
-              const tables = await config.api.table.fetch()
+        const tables = await config.api.table.fetch()
-              expect(tables).toHaveLength(2)
+        expect(tables).toHaveLength(2)
 
-              tables.sort((a, b) => a.name.localeCompare(b.name))
+        tables.sort((a, b) => a.name.localeCompare(b.name))
-              const [agencyProjects, users] = tables
+        const [agencyProjects, users] = tables
-              expect(agencyProjects.name).toBe("Agency Projects")
+        expect(agencyProjects.name).toBe("Agency Projects")
-              expect(users.name).toBe("Users")
+        expect(users.name).toBe("Users")
 
-              const { rows } = await config.api.row.search(
+        const { rows } = await config.api.row.search(agencyProjects._id!, {
-                agencyProjects._id!,
+          tableId: agencyProjects._id!,
-                {
+          query: {},
-                  tableId: agencyProjects._id!,
+        })
-                  query: {},
-                }
-              )
 
-              expect(rows).toHaveLength(3)
+        expect(rows).toHaveLength(3)
-            })
+      })
-          }
+    })
-        )
-      }
-    )
   })
 })

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -1,10 +1,6 @@
 import * as setup from "../../../api/routes/tests/utilities"
 import { basicTable } from "../../../tests/utilities/structures"
-import {
+import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
-  db as dbCore,
-  features,
-  SQLITE_DESIGN_DOC_ID,
-} from "@budibase/backend-core"
 import {
   LinkDocument,
   DocumentType,
@@ -70,24 +66,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
   }
 }
 
-async function sqsDisabled(cb: () => Promise<void>) {
-  await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
-}
-
-async function sqsEnabled(cb: () => Promise<void>) {
-  await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
-}
-
 describe("SQS migration", () => {
   beforeAll(async () => {
-    await sqsDisabled(async () => {
+    await config.init()
-      await config.init()
+    const table = await config.api.table.save(basicTable())
-      const table = await config.api.table.save(basicTable())
+    tableId = table._id!
-      tableId = table._id!
+    const db = dbCore.getDB(config.appId!)
-      const db = dbCore.getDB(config.appId!)
+    // old link document
-      // old link document
+    await db.put(oldLinkDocument())
-      await db.put(oldLinkDocument())
-    })
   })
 
   beforeEach(async () => {
@@ -101,43 +87,32 @@ describe("SQS migration", () => {
 
   it("test migration runs as expected against an older DB", async () => {
     const db = dbCore.getDB(config.appId!)
-    // confirm nothing exists initially
+
-    await sqsDisabled(async () => {
+    // remove sqlite design doc to simulate it comes from an older installation
-      let error: any | undefined
+    const doc = await db.get(SQLITE_DESIGN_DOC_ID)
-      try {
+    await db.remove({ _id: doc._id, _rev: doc._rev })
-        await db.get(SQLITE_DESIGN_DOC_ID)
+
-      } catch (err: any) {
+    await processMigrations(config.appId!, MIGRATIONS)
-        error = err
+    const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
-      }
+    expect(designDoc.sql.tables).toBeDefined()
-      expect(error).toBeDefined()
+    const mainTableDef = designDoc.sql.tables[tableId]
-      expect(error.status).toBe(404)
+    expect(mainTableDef).toBeDefined()
+    expect(mainTableDef.fields[prefix("name")]).toEqual({
+      field: "name",
+      type: SQLiteType.TEXT,
+    })
+    expect(mainTableDef.fields[prefix("description")]).toEqual({
+      field: "description",
+      type: SQLiteType.TEXT,
     })
 
-    await sqsEnabled(async () => {
+    const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
-      await processMigrations(config.appId!, MIGRATIONS)
+    const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
-      const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+    expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2))
-      expect(designDoc.sql.tables).toBeDefined()
+    // should have swapped the documents
-      const mainTableDef = designDoc.sql.tables[tableId]
+    expect(linkDoc.doc1.tableId).toEqual(tableId2)
-      expect(mainTableDef).toBeDefined()
+    expect(linkDoc.doc1.rowId).toEqual(rowId2)
-      expect(mainTableDef.fields[prefix("name")]).toEqual({
+    expect(linkDoc.doc2.tableId).toEqual(tableId1)
-        field: "name",
+    expect(linkDoc.doc2.rowId).toEqual(rowId1)
-        type: SQLiteType.TEXT,
-      })
-      expect(mainTableDef.fields[prefix("description")]).toEqual({
-        field: "description",
-        type: SQLiteType.TEXT,
-      })
-
-      const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
-      const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
-      expect(linkDoc.tableId).toEqual(
-        generateJunctionTableID(tableId1, tableId2)
-      )
-      // should have swapped the documents
-      expect(linkDoc.doc1.tableId).toEqual(tableId2)
-      expect(linkDoc.doc1.rowId).toEqual(rowId2)
-      expect(linkDoc.doc2.tableId).toEqual(tableId1)
-      expect(linkDoc.doc2.rowId).toEqual(rowId1)
-    })
   })
 })

packages/server/src/db/linkedRows/index.ts

@@ -14,11 +14,10 @@ import {
   coreOutputProcessing,
   processFormulas,
 } from "../../utilities/rowProcessor"
-import { context, features } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import {
   ContextUser,
   EventType,
-  FeatureFlag,
   FieldType,
   LinkDocumentValue,
   Row,
@@ -251,19 +250,13 @@ export async function squashLinks<T = Row[] | Row>(
   source: Table | ViewV2,
   enriched: T
 ): Promise<T> {
-  const allowRelationshipSchemas = await features.flags.isEnabled(
-    FeatureFlag.ENRICHED_RELATIONSHIPS
-  )
-
   let viewSchema: ViewV2Schema = {}
   if (sdk.views.isView(source)) {
     if (helpers.views.isCalculationView(source)) {
       return enriched
     }
 
-    if (allowRelationshipSchemas) {
+    viewSchema = source.schema || {}
-      viewSchema = source.schema || {}
-    }
   }
 
   let table: Table

packages/server/src/sdk/app/rows/search.ts

@@ -1,11 +1,8 @@
 import {
   EmptyFilterOption,
-  FeatureFlag,
   LegacyFilter,
-  LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilterKey,
   SearchFilters,
   SearchResponse,
   SortOrder,
@@ -19,7 +16,6 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
 import { checkFilters, searchInputMapping } from "./search/utils"
-import { db, features } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
 import { enrichSearchContext } from "../../../api/controllers/row/utils"
@@ -104,44 +100,14 @@ export async function search(
       }
       viewQuery = checkFilters(table, viewQuery)
 
-      const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
+      const conditions = viewQuery ? [viewQuery] : []
-      const supportsLogicalOperators =
+      options.query = {
-        isExternalTableID(view.tableId) || sqsEnabled
+        $and: {
-
+          conditions: [...conditions, options.query],
-      if (!supportsLogicalOperators) {
+        },
-        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+      }
-        // It needs to be ignored entirely
+      if (viewQuery.onEmptyFilter) {
-        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+        options.query.onEmptyFilter = viewQuery.onEmptyFilter
-          ? view.query
-          : []
-
-        const { filters } = dataFilters.splitFiltersArray(queryFilters)
-
-        // Extract existing fields
-        const existingFields = filters.map(filter =>
-          db.removeKeyNumbering(filter.field)
-        )
-
-        // Carry over filters for unused fields
-        Object.keys(options.query).forEach(key => {
-          const operator = key as Exclude<SearchFilterKey, LogicalOperator>
-          Object.keys(options.query[operator] || {}).forEach(field => {
-            if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              viewQuery[operator]![field] = options.query[operator]![field]
-            }
-          })
-        })
-        options.query = viewQuery
-      } else {
-        const conditions = viewQuery ? [viewQuery] : []
-        options.query = {
-          $and: {
-            conditions: [...conditions, options.query],
-          },
-        }
-        if (viewQuery.onEmptyFilter) {
-          options.query.onEmptyFilter = viewQuery.onEmptyFilter
-        }
       }
     }
 
@@ -170,12 +136,9 @@ export async function search(
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })
       result = await external.search(options, source)
-    } else if (await features.flags.isEnabled(FeatureFlag.SQS)) {
+    } else {
       span?.addTags({ searchType: "sqs" })
       result = await internal.sqs.search(options, source)
-    } else {
-      span?.addTags({ searchType: "lucene" })
-      result = await internal.lucene.search(options, source)
     }
 
     span.addTags({

packages/server/src/sdk/app/rows/search/internal/index.ts

@@ -1,3 +1,2 @@
 export * as sqs from "./sqs"
-export * as lucene from "./lucene"
 export * from "./internal"

packages/server/src/sdk/app/rows/search/internal/lucene.ts

@@ -1,79 +0,0 @@
-import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
-import { fullSearch, paginatedSearch } from "../utils"
-import { InternalTables } from "../../../../../db/utils"
-import {
-  Row,
-  RowSearchParams,
-  SearchResponse,
-  SortType,
-  Table,
-  User,
-  ViewV2,
-} from "@budibase/types"
-import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
-import { outputProcessing } from "../../../../../utilities/rowProcessor"
-import pick from "lodash/pick"
-import sdk from "../../../../"
-
-export async function search(
-  options: RowSearchParams,
-  source: Table | ViewV2
-): Promise<SearchResponse<Row>> {
-  let table: Table
-  if (sdk.views.isView(source)) {
-    table = await sdk.views.getTable(source.id)
-  } else {
-    table = source
-  }
-
-  const { paginate, query } = options
-
-  const params: RowSearchParams = {
-    tableId: options.tableId,
-    viewId: options.viewId,
-    sort: options.sort,
-    sortOrder: options.sortOrder,
-    sortType: options.sortType,
-    limit: options.limit,
-    bookmark: options.bookmark,
-    version: options.version,
-    disableEscaping: options.disableEscaping,
-    query: {},
-  }
-
-  if (params.sort && !params.sortType) {
-    const schema = table.schema
-    const sortField = schema[params.sort]
-    params.sortType =
-      sortField.type === "number" ? SortType.NUMBER : SortType.STRING
-  }
-
-  let response
-  if (paginate) {
-    response = await paginatedSearch(query, params)
-  } else {
-    response = await fullSearch(query, params)
-  }
-
-  // Enrich search results with relationships
-  if (response.rows && response.rows.length) {
-    // enrich with global users if from users table
-    if (table._id === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
-    }
-
-    const visibleFields =
-      options.fields ||
-      Object.keys(source.schema || {}).filter(
-        key => source.schema?.[key].visible !== false
-      )
-    const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
-    response.rows = response.rows.map((r: any) => pick(r, allowedFields))
-
-    response.rows = await outputProcessing(source, response.rows, {
-      squash: true,
-    })
-  }
-
-  return response
-}

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -10,7 +10,7 @@ import {
 import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
 import { search } from "../../../../../sdk/app/rows/search"
 import { generator } from "@budibase/backend-core/tests"
-import { features } from "@budibase/backend-core"
+
 import {
   DatabaseName,
   getDatasource,
@@ -21,30 +21,20 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures"
 // (e.g. limiting searches to returning specific fields). If it's possible to
 // test through the API, it should be done there instead.
 describe.each([
-  ["lucene", undefined],
+  ["internal", undefined],
-  ["sqs", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("search sdk (%s)", (name, dsProvider) => {
-  const isSqs = name === "sqs"
+  const isInternal = name === "internal"
-  const isLucene = name === "lucene"
-  const isInternal = isLucene || isSqs
   const config = new TestConfiguration()
 
-  let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let table: Table
 
   beforeAll(async () => {
-    await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
+    await config.init()
-      config.init()
-    )
-
-    envCleanup = features.testutils.setFeatureFlags("*", {
-      SQS: isSqs,
-    })
 
     if (dsProvider) {
       datasource = await config.createDatasource({
@@ -105,9 +95,6 @@ describe.each([
 
   afterAll(async () => {
     config.end()
-    if (envCleanup) {
-      envCleanup()
-    }
   })
 
   it("querying by fields will always return data attribute columns", async () => {
@@ -211,36 +198,35 @@ describe.each([
     })
   })
 
-  !isLucene &&
+  it.each([
-    it.each([
+    [["id", "name", "age"], 3],
-      [["id", "name", "age"], 3],
+    [["name", "age"], 10],
-      [["name", "age"], 10],
+  ])(
-    ])(
+    "cannot query by non search fields (fields: %s)",
-      "cannot query by non search fields (fields: %s)",
+    async (queryFields, expectedRows) => {
-      async (queryFields, expectedRows) => {
+      await config.doInContext(config.appId, async () => {
-        await config.doInContext(config.appId, async () => {
+        const { rows } = await search({
-          const { rows } = await search({
+          tableId: table._id!,
-            tableId: table._id!,
+          query: {
-            query: {
+            $or: {
-              $or: {
+              conditions: [
-                conditions: [
+                {
-                  {
+                  $and: {
-                    $and: {
+                    conditions: [
-                      conditions: [
+                      { range: { id: { low: 2, high: 4 } } },
-                        { range: { id: { low: 2, high: 4 } } },
+                      { range: { id: { low: 3, high: 5 } } },
-                        { range: { id: { low: 3, high: 5 } } },
+                    ],
-                      ],
-                    },
                   },
-                  { equal: { id: 7 } },
+                },
-                ],
+                { equal: { id: 7 } },
-              },
+              ],
             },
-            fields: queryFields,
+          },
-          })
+          fields: queryFields,
-
-          expect(rows).toHaveLength(expectedRows)
         })
-      }
+
-    )
+        expect(rows).toHaveLength(expectedRows)
+      })
+    }
+  )
 })

packages/server/src/sdk/app/tables/getters.ts

@@ -1,4 +1,4 @@
-import { context, features } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
@@ -12,7 +12,6 @@ import {
   TableResponse,
   TableSourceType,
   TableViewsResponse,
-  FeatureFlag,
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
@@ -49,10 +48,7 @@ export async function processTable(table: Table): Promise<Table> {
       type: "table",
       sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
-    }
+      sql: true,
-    const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
-    if (sqsEnabled) {
-      processed.sql = true
     }
     return processed
   }

packages/server/src/utilities/rowProcessor/index.ts

@@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils"
 import {
   cache,
   context,
-  features,
   HTTPError,
   objectStore,
   utils,
@@ -19,7 +18,6 @@ import {
   Table,
   User,
   ViewV2,
-  FeatureFlag,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
@@ -423,45 +421,43 @@ export async function coreOutputProcessing(
 
   // remove null properties to match internal API
   const isExternal = isExternalTableID(table._id!)
-  if (isExternal || (await features.flags.isEnabled(FeatureFlag.SQS))) {
+  for (const row of rows) {
-    for (const row of rows) {
+    for (const key of Object.keys(row)) {
-      for (const key of Object.keys(row)) {
+      if (row[key] === null) {
-        if (row[key] === null) {
+        delete row[key]
-          delete row[key]
+      } else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
-        } else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
+        for (const link of row[key] || []) {
-          for (const link of row[key] || []) {
+          for (const linkKey of Object.keys(link)) {
-            for (const linkKey of Object.keys(link)) {
+            if (link[linkKey] === null) {
-              if (link[linkKey] === null) {
+              delete link[linkKey]
-                delete link[linkKey]
-              }
             }
           }
         }
       }
     }
+  }
 
-    if (sdk.views.isView(source)) {
+  if (sdk.views.isView(source)) {
-      // We ensure calculation fields are returned as numbers.  During the
+    // We ensure calculation fields are returned as numbers.  During the
-      // testing of this feature it was discovered that the COUNT operation
+    // testing of this feature it was discovered that the COUNT operation
-      // returns a string for MySQL, MariaDB, and Postgres. But given that all
+    // returns a string for MySQL, MariaDB, and Postgres. But given that all
-      // calculation fields (except ones operating on BIGINTs) should be
+    // calculation fields (except ones operating on BIGINTs) should be
-      // numbers, we blanket make sure of that here.
+    // numbers, we blanket make sure of that here.
-      for (const [name, field] of Object.entries(
+    for (const [name, field] of Object.entries(
-        helpers.views.calculationFields(source)
+      helpers.views.calculationFields(source)
-      )) {
+    )) {
-        if ("field" in field) {
+      if ("field" in field) {
-          const targetSchema = table.schema[field.field]
+        const targetSchema = table.schema[field.field]
-          // We don't convert BIGINT fields to floats because we could lose
+        // We don't convert BIGINT fields to floats because we could lose
-          // precision.
+        // precision.
-          if (targetSchema.type === FieldType.BIGINT) {
+        if (targetSchema.type === FieldType.BIGINT) {
-            continue
+          continue
-          }
         }
+      }
 
-        for (const row of rows) {
+      for (const row of rows) {
-          if (typeof row[name] === "string") {
+        if (typeof row[name] === "string") {
-            row[name] = parseFloat(row[name])
+          row[name] = parseFloat(row[name])
-          }
         }
       }
     }

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { outputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
-import { features } from "@budibase/backend-core"
+
 import * as bbReferenceProcessor from "../bbReferenceProcessor"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -21,7 +21,6 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
 
 describe("rowProcessor - outputProcessing", () => {
   const config = new TestConfiguration()
-  let cleanupFlags: () => void = () => {}
 
   beforeAll(async () => {
     await config.init()
@@ -33,11 +32,6 @@ describe("rowProcessor - outputProcessing", () => {
 
   beforeEach(() => {
     jest.resetAllMocks()
-    cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
-  })
-
-  afterEach(() => {
-    cleanupFlags()
   })
 
   const processOutputBBReferenceMock =

packages/shared-core/src/filters.ts

@@ -527,7 +527,12 @@ export function search<T extends Record<string, any>>(
 ): SearchResponse<T> {
   let result = runQuery(docs, query.query)
   if (query.sort) {
-    result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
+    result = sort(
+      result,
+      query.sort,
+      query.sortOrder || SortOrder.ASCENDING,
+      query.sortType
+    )
   }
   const totalRows = result.length
   if (query.limit) {

packages/types/src/sdk/db.ts

@@ -12,7 +12,6 @@ import type PouchDB from "pouchdb-find"
 
 export enum SearchIndex {
   ROWS = "rows",
-  AUDIT = "audit",
   USER = "user",
 }
 

packages/types/src/sdk/featureFlag.ts

@@ -2,10 +2,9 @@ export enum FeatureFlag {
   PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE",
   PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
   AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
-  SQS = "SQS",
   AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
   DEFAULT_VALUES = "DEFAULT_VALUES",
-  ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",
+
   BUDIBASE_AI = "BUDIBASE_AI",
 }
 

packages/worker/src/api/controllers/system/environment.ts

@@ -1,6 +1,6 @@
-import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
+import { Ctx, MaintenanceType } from "@budibase/types"
 import env from "../../../environment"
-import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
+import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
 import { helpers } from "@budibase/shared-core"
 
@@ -35,10 +35,7 @@ async function isSqsAvailable() {
 }
 
 async function isSqsMissing() {
-  return (
+  return !(await isSqsAvailable())
-    (await features.flags.isEnabled(FeatureFlag.SQS)) &&
-    !(await isSqsAvailable())
-  )
 }
 
 export const fetch = async (ctx: Ctx) => {

packages/worker/src/api/routes/global/tests/auditLogs.spec.ts

@@ -1,5 +1,5 @@
 import { mocks, structures } from "@budibase/backend-core/tests"
-import { context, events, features } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
 import { Event, IdentityType } from "@budibase/types"
 import { TestConfiguration } from "../../../../tests"
 
@@ -12,19 +12,14 @@ const BASE_IDENTITY = {
 const USER_AUDIT_LOG_COUNT = 3
 const APP_ID = "app_1"
 
-describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
+describe("/api/global/auditlogs (%s)", () => {
   const config = new TestConfiguration()
-  let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    envCleanup = features.testutils.setFeatureFlags("*", {
-      SQS: method === "sql",
-    })
     await config.beforeAll()
   })
 
   afterAll(async () => {
-    envCleanup?.()
     await config.afterAll()
   })