Merge pull request #14835 from Budibase/chore/sqs-always-on

Remove SQS flag
This commit is contained in:
Adria Navarro 2024-11-18 10:53:23 +01:00 committed by GitHub
commit 36f93dd452
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 2895 additions and 3396 deletions

View File

@ -10,7 +10,6 @@ import {
DatabaseQueryOpts,
DBError,
Document,
FeatureFlag,
isDocument,
RowResponse,
RowValue,
@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
import { DDInstrumentedDatabase } from "../instrumentation"
import { checkSlashesInUrl } from "../../helpers"
import { sqlLog } from "../../sql/utils"
import { flags } from "../../features"
const DATABASE_NOT_FOUND = "Database does not exist."
@ -456,10 +454,7 @@ export class DatabaseImpl implements Database {
}
async destroy() {
if (
(await flags.isEnabled(FeatureFlag.SQS)) &&
(await this.exists(SQLITE_DESIGN_DOC_ID))
) {
if (await this.exists(SQLITE_DESIGN_DOC_ID)) {
// delete the design document, then run the cleanup operation
const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
// remove all tables - save the definition then trigger a cleanup

View File

@ -269,7 +269,6 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({
[FeatureFlag.DEFAULT_VALUES]: Flag.boolean(true),
[FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(true),
[FeatureFlag.SQS]: Flag.boolean(true),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(true),
[FeatureFlag.BUDIBASE_AI]: Flag.boolean(true),

@ -1 +1 @@
Subproject commit 80770215c6159e4d47f3529fd02e74bc8ad07543
Subproject commit a56696a4af5667617746600fc75fe6a01744b692

View File

@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils"
import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { context, events, features, HTTPError } from "@budibase/backend-core"
import { context, events, HTTPError } from "@budibase/backend-core"
import {
AutoFieldSubType,
Database,
Datasource,
FeatureFlag,
FieldSchema,
FieldType,
NumberFieldMetadata,
@ -336,9 +335,8 @@ class TableSaveFunctions {
importRows: this.importRows,
userId: this.userId,
})
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.addTable(table)
}
await sdk.tables.sqs.addTable(table)
return table
}
@ -530,9 +528,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
if (rows) {
await AttachmentCleanup.tableDelete(table, rows)
}
if (await features.flags.isEnabled(FeatureFlag.SQS)) {
await sdk.tables.sqs.removeTable(table)
}
await sdk.tables.sqs.removeTable(table)
}
const _TableSaveFunctions = TableSaveFunctions

View File

@ -16,7 +16,7 @@ jest.mock("../../../utilities/redis", () => ({
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import { AppStatus } from "../../../db/utils"
import { events, utils, context, features } from "@budibase/backend-core"
import { events, utils, context } from "@budibase/backend-core"
import env from "../../../environment"
import { type App, BuiltinPermissionID } from "@budibase/types"
import tk from "timekeeper"
@ -355,21 +355,6 @@ describe("/applications", () => {
expect(events.app.deleted).toHaveBeenCalledTimes(1)
expect(events.app.unpublished).toHaveBeenCalledTimes(1)
})
it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => {
const prodAppId = app.appId.replace("_dev", "")
nock("http://localhost:10000")
.delete(`/api/global/roles/${prodAppId}`)
.reply(200, {})
await features.testutils.withFeatureFlags(
"*",
{ SQS: true },
async () => {
await config.api.application.delete(app.appId)
}
)
})
})
describe("POST /api/applications/:appId/duplicate", () => {

View File

@ -19,17 +19,14 @@ import {
import { quotas } from "@budibase/pro"
import {
AIOperationEnum,
AttachmentFieldMetadata,
AutoFieldSubType,
Datasource,
DateFieldMetadata,
DeleteRow,
FieldSchema,
FieldType,
BBReferenceFieldSubType,
FormulaType,
INTERNAL_TABLE_SOURCE_ID,
NumberFieldMetadata,
QuotaUsageType,
RelationshipType,
Row,
@ -90,8 +87,7 @@ async function waitForEvent(
}
describe.each([
["lucene", undefined],
["sqs", undefined],
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
@ -99,8 +95,6 @@ describe.each([
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/rows (%s)", (providerType, dsProvider) => {
const isInternal = dsProvider === undefined
const isLucene = providerType === "lucene"
const isSqs = providerType === "sqs"
const isMSSQL = providerType === DatabaseName.SQL_SERVER
const isOracle = providerType === DatabaseName.ORACLE
const config = setup.getConfig()
@ -108,15 +102,9 @@ describe.each([
let table: Table
let datasource: Datasource | undefined
let client: Knex | undefined
let envCleanup: (() => void) | undefined
beforeAll(async () => {
await features.testutils.withFeatureFlags("*", { SQS: true }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
await config.init()
if (dsProvider) {
const rawDatasource = await dsProvider
@ -129,9 +117,6 @@ describe.each([
afterAll(async () => {
setup.afterAll()
if (envCleanup) {
envCleanup()
}
})
function saveTableRequest(
@ -381,185 +366,6 @@ describe.each([
expect(ids).toEqual(expect.arrayContaining(sequence))
})
isLucene &&
it("row values are coerced", async () => {
const str: FieldSchema = {
type: FieldType.STRING,
name: "str",
constraints: { type: "string", presence: false },
}
const singleAttachment: FieldSchema = {
type: FieldType.ATTACHMENT_SINGLE,
name: "single attachment",
constraints: { presence: false },
}
const attachmentList: AttachmentFieldMetadata = {
type: FieldType.ATTACHMENTS,
name: "attachments",
constraints: { type: "array", presence: false },
}
const signature: FieldSchema = {
type: FieldType.SIGNATURE_SINGLE,
name: "signature",
constraints: { presence: false },
}
const bool: FieldSchema = {
type: FieldType.BOOLEAN,
name: "boolean",
constraints: { type: "boolean", presence: false },
}
const number: NumberFieldMetadata = {
type: FieldType.NUMBER,
name: "str",
constraints: { type: "number", presence: false },
}
const datetime: DateFieldMetadata = {
type: FieldType.DATETIME,
name: "datetime",
constraints: {
type: "string",
presence: false,
datetime: { earliest: "", latest: "" },
},
}
const arrayField: FieldSchema = {
type: FieldType.ARRAY,
constraints: {
type: JsonFieldSubType.ARRAY,
presence: false,
inclusion: ["One", "Two", "Three"],
},
name: "Sample Tags",
sortable: false,
}
const optsField: FieldSchema = {
name: "Sample Opts",
type: FieldType.OPTIONS,
constraints: {
type: "string",
presence: false,
inclusion: ["Alpha", "Beta", "Gamma"],
},
}
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: str,
stringUndefined: str,
stringNull: str,
stringString: str,
numberEmptyString: number,
numberNull: number,
numberUndefined: number,
numberString: number,
numberNumber: number,
datetimeEmptyString: datetime,
datetimeNull: datetime,
datetimeUndefined: datetime,
datetimeString: datetime,
datetimeDate: datetime,
boolNull: bool,
boolEmpty: bool,
boolUndefined: bool,
boolString: bool,
boolBool: bool,
singleAttachmentNull: singleAttachment,
singleAttachmentUndefined: singleAttachment,
attachmentListNull: attachmentList,
attachmentListUndefined: attachmentList,
attachmentListEmpty: attachmentList,
attachmentListEmptyArrayStr: attachmentList,
signatureNull: signature,
signatureUndefined: signature,
arrayFieldEmptyArrayStr: arrayField,
arrayFieldArrayStrKnown: arrayField,
arrayFieldNull: arrayField,
arrayFieldUndefined: arrayField,
optsFieldEmptyStr: optsField,
optsFieldUndefined: optsField,
optsFieldNull: optsField,
optsFieldStrKnown: optsField,
},
})
)
const datetimeStr = "1984-04-20T00:00:00.000Z"
const row = await config.api.row.save(table._id!, {
name: "Test Row",
stringUndefined: undefined,
stringNull: null,
stringString: "i am a string",
numberEmptyString: "",
numberNull: null,
numberUndefined: undefined,
numberString: "123",
numberNumber: 123,
datetimeEmptyString: "",
datetimeNull: null,
datetimeUndefined: undefined,
datetimeString: datetimeStr,
datetimeDate: new Date(datetimeStr),
boolNull: null,
boolEmpty: "",
boolUndefined: undefined,
boolString: "true",
boolBool: true,
tableId: table._id,
singleAttachmentNull: null,
singleAttachmentUndefined: undefined,
attachmentListNull: null,
attachmentListUndefined: undefined,
attachmentListEmpty: "",
attachmentListEmptyArrayStr: "[]",
signatureNull: null,
signatureUndefined: undefined,
arrayFieldEmptyArrayStr: "[]",
arrayFieldUndefined: undefined,
arrayFieldNull: null,
arrayFieldArrayStrKnown: "['One']",
optsFieldEmptyStr: "",
optsFieldUndefined: undefined,
optsFieldNull: null,
optsFieldStrKnown: "Alpha",
})
expect(row.stringUndefined).toBe(undefined)
expect(row.stringNull).toBe(null)
expect(row.stringString).toBe("i am a string")
expect(row.numberEmptyString).toBe(null)
expect(row.numberNull).toBe(null)
expect(row.numberUndefined).toBe(undefined)
expect(row.numberString).toBe(123)
expect(row.numberNumber).toBe(123)
expect(row.datetimeEmptyString).toBe(null)
expect(row.datetimeNull).toBe(null)
expect(row.datetimeUndefined).toBe(undefined)
expect(row.datetimeString).toBe(new Date(datetimeStr).toISOString())
expect(row.datetimeDate).toBe(new Date(datetimeStr).toISOString())
expect(row.boolNull).toBe(null)
expect(row.boolEmpty).toBe(null)
expect(row.boolUndefined).toBe(undefined)
expect(row.boolString).toBe(true)
expect(row.boolBool).toBe(true)
expect(row.singleAttachmentNull).toEqual(null)
expect(row.singleAttachmentUndefined).toBe(undefined)
expect(row.attachmentListNull).toEqual([])
expect(row.attachmentListUndefined).toBe(undefined)
expect(row.attachmentListEmpty).toEqual([])
expect(row.attachmentListEmptyArrayStr).toEqual([])
expect(row.signatureNull).toEqual(null)
expect(row.signatureUndefined).toBe(undefined)
expect(row.arrayFieldEmptyArrayStr).toEqual([])
expect(row.arrayFieldNull).toEqual([])
expect(row.arrayFieldUndefined).toEqual(undefined)
expect(row.optsFieldEmptyStr).toEqual(null)
expect(row.optsFieldUndefined).toEqual(undefined)
expect(row.optsFieldNull).toEqual(null)
expect(row.arrayFieldArrayStrKnown).toEqual(["One"])
expect(row.optsFieldStrKnown).toEqual("Alpha")
})
isInternal &&
it("doesn't allow creating in user table", async () => {
const response = await config.api.row.save(
@ -1023,104 +829,103 @@ describe.each([
})
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can create rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
},
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: [
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
],
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
})
it("can create rows with no relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
})
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
it("can create rows with only one relationships field", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can create rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: [
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
],
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
})
it("can create rows with no relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
})
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
it("can create rows with only one relationships field", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [relatedRows[1]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
})
})
describe("get", () => {
@ -1224,133 +1029,132 @@ describe.each([
expect(rows).toHaveLength(1)
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can edit rows with both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
},
})
row = await config.api.row.save(table._id!, {
...row,
related1: [relatedRows[0]._id!, relatedRows[1]._id!],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: expect.arrayContaining([
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
]),
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
})
it("can drop existing relationship", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
it("can drop both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
})
)
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can edit rows with both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [relatedRows[0]._id!, relatedRows[1]._id!],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related1: expect.arrayContaining([
{
_id: relatedRows[0]._id,
primaryDisplay: relatedRows[0].name,
},
{
_id: relatedRows[1]._id,
primaryDisplay: relatedRows[1].name,
},
]),
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
})
it("can drop existing relationship", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [relatedRows[2]._id!],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
related2: [
{
_id: relatedRows[2]._id,
primaryDisplay: relatedRows[2].name,
},
],
})
)
expect(row.related1).toBeUndefined()
})
it("can drop both relationships", async () => {
let row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
row = await config.api.row.save(table._id!, {
...row,
related1: [],
related2: [],
})
expect(row).toEqual(
expect.objectContaining({
name: "test",
})
)
expect(row.related1).toBeUndefined()
expect(row.related2).toBeUndefined()
})
})
})
describe("patch", () => {
@ -1628,72 +1432,71 @@ describe.each([
expect(res.length).toEqual(2)
})
!isLucene &&
describe("relations to same table", () => {
let relatedRows: Row[]
describe("relations to same table", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
},
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can delete rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
beforeAll(async () => {
const relatedTable = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
},
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
it("can delete rows with empty relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [],
)
const relatedTableId = relatedTable._id!
table = await config.api.table.save(
defaultTable({
schema: {
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
related2: {
type: FieldType.LINK,
name: "related2",
fieldName: "main2",
tableId: relatedTableId,
relationshipType: RelationshipType.MANY_TO_MANY,
},
},
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
)
relatedRows = await Promise.all([
config.api.row.save(relatedTableId, { name: "foo" }),
config.api.row.save(relatedTableId, { name: "bar" }),
config.api.row.save(relatedTableId, { name: "baz" }),
config.api.row.save(relatedTableId, { name: "boo" }),
])
})
it("can delete rows with both relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!],
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
it("can delete rows with empty relationships", async () => {
const row = await config.api.row.save(table._id!, {
name: "test",
related1: [],
related2: [],
})
await config.api.row.delete(table._id!, { _id: row._id! })
await config.api.row.get(table._id!, row._id!, { status: 404 })
})
})
})
describe("validate", () => {
@ -3422,7 +3225,7 @@ describe.each([
)
})
isSqs &&
isInternal &&
describe("AI fields", () => {
let table: Table

File diff suppressed because it is too large Load Diff

View File

@ -2,7 +2,6 @@ import * as setup from "./utilities"
import path from "path"
import nock from "nock"
import { generator } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
interface App {
background: string
@ -82,48 +81,36 @@ describe("/templates", () => {
})
describe("create app from template", () => {
it.each(["sqs", "lucene"])(
`should be able to create an app from a template (%s)`,
async source => {
await features.testutils.withFeatureFlags(
"*",
{ SQS: source === "sqs" },
async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
it("should be able to create an app from a template", async () => {
const name = generator.guid().replaceAll("-", "")
const url = `/${name}`
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
const app = await config.api.application.create({
name,
url,
useTemplate: "true",
templateName: "Agency Client Portal",
templateKey: "app/agency-client-portal",
})
expect(app.name).toBe(name)
expect(app.url).toBe(url)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
await config.withApp(app, async () => {
const tables = await config.api.table.fetch()
expect(tables).toHaveLength(2)
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
tables.sort((a, b) => a.name.localeCompare(b.name))
const [agencyProjects, users] = tables
expect(agencyProjects.name).toBe("Agency Projects")
expect(users.name).toBe("Users")
const { rows } = await config.api.row.search(
agencyProjects._id!,
{
tableId: agencyProjects._id!,
query: {},
}
)
const { rows } = await config.api.row.search(agencyProjects._id!, {
tableId: agencyProjects._id!,
query: {},
})
expect(rows).toHaveLength(3)
})
}
)
}
)
expect(rows).toHaveLength(3)
})
})
})
})

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +1,6 @@
import * as setup from "../../../api/routes/tests/utilities"
import { basicTable } from "../../../tests/utilities/structures"
import {
db as dbCore,
features,
SQLITE_DESIGN_DOC_ID,
} from "@budibase/backend-core"
import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
import {
LinkDocument,
DocumentType,
@ -70,24 +66,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
}
async function sqsDisabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: false }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await features.testutils.withFeatureFlags("*", { SQS: true }, cb)
}
describe("SQS migration", () => {
beforeAll(async () => {
await sqsDisabled(async () => {
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
beforeEach(async () => {
@ -101,43 +87,32 @@ describe("SQS migration", () => {
it("test migration runs as expected against an older DB", async () => {
const db = dbCore.getDB(config.appId!)
// confirm nothing exists initially
await sqsDisabled(async () => {
let error: any | undefined
try {
await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err: any) {
error = err
}
expect(error).toBeDefined()
expect(error.status).toBe(404)
// remove sqlite design doc to simulate it comes from an older installation
const doc = await db.get(SQLITE_DESIGN_DOC_ID)
await db.remove({ _id: doc._id, _rev: doc._rev })
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
await sqsEnabled(async () => {
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(
generateJunctionTableID(tableId1, tableId2)
)
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2))
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
})

View File

@ -1,11 +1,8 @@
import {
EmptyFilterOption,
FeatureFlag,
LegacyFilter,
LogicalOperator,
Row,
RowSearchParams,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
@ -19,7 +16,6 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { checkFilters, searchInputMapping } from "./search/utils"
import { db, features } from "@budibase/backend-core"
import tracer from "dd-trace"
import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
import { enrichSearchContext } from "../../../api/controllers/row/utils"
@ -104,44 +100,14 @@ export async function search(
}
viewQuery = checkFilters(table, viewQuery)
const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
const supportsLogicalOperators =
isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
const { filters } = dataFilters.splitFiltersArray(queryFilters)
// Extract existing fields
const existingFields = filters.map(filter =>
db.removeKeyNumbering(filter.field)
)
// Carry over filters for unused fields
Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
viewQuery[operator]![field] = options.query[operator]![field]
}
})
})
options.query = viewQuery
} else {
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [...conditions, options.query],
},
}
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
}
@ -170,12 +136,9 @@ export async function search(
if (isExternalTable) {
span?.addTags({ searchType: "external" })
result = await external.search(options, source)
} else if (await features.flags.isEnabled(FeatureFlag.SQS)) {
} else {
span?.addTags({ searchType: "sqs" })
result = await internal.sqs.search(options, source)
} else {
span?.addTags({ searchType: "lucene" })
result = await internal.lucene.search(options, source)
}
span.addTags({

View File

@ -1,3 +1,2 @@
export * as sqs from "./sqs"
export * as lucene from "./lucene"
export * from "./internal"

View File

@ -1,79 +0,0 @@
import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
import { fullSearch, paginatedSearch } from "../utils"
import { InternalTables } from "../../../../../db/utils"
import {
Row,
RowSearchParams,
SearchResponse,
SortType,
Table,
User,
ViewV2,
} from "@budibase/types"
import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
import { outputProcessing } from "../../../../../utilities/rowProcessor"
import pick from "lodash/pick"
import sdk from "../../../../"
export async function search(
options: RowSearchParams,
source: Table | ViewV2
): Promise<SearchResponse<Row>> {
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
const { paginate, query } = options
const params: RowSearchParams = {
tableId: options.tableId,
viewId: options.viewId,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
limit: options.limit,
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
query: {},
}
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
params.sortType =
sortField.type === "number" ? SortType.NUMBER : SortType.STRING
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (table._id === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, allowedFields))
response.rows = await outputProcessing(source, response.rows, {
squash: true,
})
}
return response
}

View File

@ -10,7 +10,7 @@ import {
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { search } from "../../../../../sdk/app/rows/search"
import { generator } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
import {
DatabaseName,
getDatasource,
@ -21,30 +21,20 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures"
// (e.g. limiting searches to returning specific fields). If it's possible to
// test through the API, it should be done there instead.
describe.each([
["lucene", undefined],
["sqs", undefined],
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("search sdk (%s)", (name, dsProvider) => {
const isSqs = name === "sqs"
const isLucene = name === "lucene"
const isInternal = isLucene || isSqs
const isInternal = name === "internal"
const config = new TestConfiguration()
let envCleanup: (() => void) | undefined
let datasource: Datasource | undefined
let table: Table
beforeAll(async () => {
await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () =>
config.init()
)
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: isSqs,
})
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
@ -105,9 +95,6 @@ describe.each([
afterAll(async () => {
config.end()
if (envCleanup) {
envCleanup()
}
})
it("querying by fields will always return data attribute columns", async () => {
@ -211,36 +198,35 @@ describe.each([
})
})
!isLucene &&
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
it.each([
[["id", "name", "age"], 3],
[["name", "age"], 10],
])(
"cannot query by non search fields (fields: %s)",
async (queryFields, expectedRows) => {
await config.doInContext(config.appId, async () => {
const { rows } = await search({
tableId: table._id!,
query: {
$or: {
conditions: [
{
$and: {
conditions: [
{ range: { id: { low: 2, high: 4 } } },
{ range: { id: { low: 3, high: 5 } } },
],
},
{ equal: { id: 7 } },
],
},
},
{ equal: { id: 7 } },
],
},
fields: queryFields,
})
expect(rows).toHaveLength(expectedRows)
},
fields: queryFields,
})
}
)
expect(rows).toHaveLength(expectedRows)
})
}
)
})

View File

@ -1,4 +1,4 @@
import { context, features } from "@budibase/backend-core"
import { context } from "@budibase/backend-core"
import { getTableParams } from "../../../db/utils"
import {
breakExternalTableId,
@ -12,7 +12,6 @@ import {
TableResponse,
TableSourceType,
TableViewsResponse,
FeatureFlag,
} from "@budibase/types"
import datasources from "../datasources"
import sdk from "../../../sdk"
@ -49,10 +48,7 @@ export async function processTable(table: Table): Promise<Table> {
type: "table",
sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
}
const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS)
if (sqsEnabled) {
processed.sql = true
sql: true,
}
return processed
}

View File

@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils"
import {
cache,
context,
features,
HTTPError,
objectStore,
utils,
@ -19,7 +18,6 @@ import {
Table,
User,
ViewV2,
FeatureFlag,
} from "@budibase/types"
import { cloneDeep } from "lodash/fp"
import {
@ -423,45 +421,43 @@ export async function coreOutputProcessing(
// remove null properties to match internal API
const isExternal = isExternalTableID(table._id!)
if (isExternal || (await features.flags.isEnabled(FeatureFlag.SQS))) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
} else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
for (const link of row[key] || []) {
for (const linkKey of Object.keys(link)) {
if (link[linkKey] === null) {
delete link[linkKey]
}
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
} else if (row[key] && table.schema[key]?.type === FieldType.LINK) {
for (const link of row[key] || []) {
for (const linkKey of Object.keys(link)) {
if (link[linkKey] === null) {
delete link[linkKey]
}
}
}
}
}
}
if (sdk.views.isView(source)) {
// We ensure calculation fields are returned as numbers. During the
// testing of this feature it was discovered that the COUNT operation
// returns a string for MySQL, MariaDB, and Postgres. But given that all
// calculation fields (except ones operating on BIGINTs) should be
// numbers, we blanket make sure of that here.
for (const [name, field] of Object.entries(
helpers.views.calculationFields(source)
)) {
if ("field" in field) {
const targetSchema = table.schema[field.field]
// We don't convert BIGINT fields to floats because we could lose
// precision.
if (targetSchema.type === FieldType.BIGINT) {
continue
}
if (sdk.views.isView(source)) {
// We ensure calculation fields are returned as numbers. During the
// testing of this feature it was discovered that the COUNT operation
// returns a string for MySQL, MariaDB, and Postgres. But given that all
// calculation fields (except ones operating on BIGINTs) should be
// numbers, we blanket make sure of that here.
for (const [name, field] of Object.entries(
helpers.views.calculationFields(source)
)) {
if ("field" in field) {
const targetSchema = table.schema[field.field]
// We don't convert BIGINT fields to floats because we could lose
// precision.
if (targetSchema.type === FieldType.BIGINT) {
continue
}
}
for (const row of rows) {
if (typeof row[name] === "string") {
row[name] = parseFloat(row[name])
}
for (const row of rows) {
if (typeof row[name] === "string") {
row[name] = parseFloat(row[name])
}
}
}

View File

@ -8,7 +8,7 @@ import {
} from "@budibase/types"
import { outputProcessing } from ".."
import { generator, structures } from "@budibase/backend-core/tests"
import { features } from "@budibase/backend-core"
import * as bbReferenceProcessor from "../bbReferenceProcessor"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
@ -21,7 +21,6 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({
describe("rowProcessor - outputProcessing", () => {
const config = new TestConfiguration()
let cleanupFlags: () => void = () => {}
beforeAll(async () => {
await config.init()
@ -33,11 +32,6 @@ describe("rowProcessor - outputProcessing", () => {
beforeEach(() => {
jest.resetAllMocks()
cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true })
})
afterEach(() => {
cleanupFlags()
})
const processOutputBBReferenceMock =

View File

@ -527,7 +527,12 @@ export function search<T extends Record<string, any>>(
): SearchResponse<T> {
let result = runQuery(docs, query.query)
if (query.sort) {
result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING)
result = sort(
result,
query.sort,
query.sortOrder || SortOrder.ASCENDING,
query.sortType
)
}
const totalRows = result.length
if (query.limit) {

View File

@ -12,7 +12,6 @@ import type PouchDB from "pouchdb-find"
export enum SearchIndex {
ROWS = "rows",
AUDIT = "audit",
USER = "user",
}

View File

@ -2,7 +2,6 @@ export enum FeatureFlag {
PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE",
PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
SQS = "SQS",
AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
DEFAULT_VALUES = "DEFAULT_VALUES",
ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",

View File

@ -1,6 +1,6 @@
import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types"
import { Ctx, MaintenanceType } from "@budibase/types"
import env from "../../../environment"
import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core"
import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
import nodeFetch from "node-fetch"
import { helpers } from "@budibase/shared-core"
@ -35,10 +35,7 @@ async function isSqsAvailable() {
}
async function isSqsMissing() {
return (
(await features.flags.isEnabled(FeatureFlag.SQS)) &&
!(await isSqsAvailable())
)
return !(await isSqsAvailable())
}
export const fetch = async (ctx: Ctx) => {

View File

@ -1,5 +1,5 @@
import { mocks, structures } from "@budibase/backend-core/tests"
import { context, events, features } from "@budibase/backend-core"
import { context, events } from "@budibase/backend-core"
import { Event, IdentityType } from "@budibase/types"
import { TestConfiguration } from "../../../../tests"
@ -12,19 +12,14 @@ const BASE_IDENTITY = {
const USER_AUDIT_LOG_COUNT = 3
const APP_ID = "app_1"
describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => {
describe("/api/global/auditlogs (%s)", () => {
const config = new TestConfiguration()
let envCleanup: (() => void) | undefined
beforeAll(async () => {
envCleanup = features.testutils.setFeatureFlags("*", {
SQS: method === "sql",
})
await config.beforeAll()
})
afterAll(async () => {
envCleanup?.()
await config.afterAll()
})