From 42f27bacb28ec27115c694802aefbe332b3a2e9b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 12:50:42 +0200 Subject: [PATCH 001/193] Cleanup SQS feature usages --- .../backend-core/src/db/couch/DatabaseImpl.ts | 7 +-- packages/pro | 2 +- .../server/src/api/controllers/table/utils.ts | 13 ++-- packages/server/src/sdk/app/tables/getters.ts | 8 +-- .../src/utilities/rowProcessor/index.ts | 62 +++++++++---------- 5 files changed, 38 insertions(+), 54 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index b807db0ee3..83b9b69d0b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -10,7 +10,6 @@ import { DatabaseQueryOpts, DBError, Document, - FeatureFlag, isDocument, RowResponse, RowValue, @@ -27,7 +26,6 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import { sqlLog } from "../../sql/utils" -import { flags } from "../../features" const DATABASE_NOT_FOUND = "Database does not exist." @@ -456,10 +454,7 @@ export class DatabaseImpl implements Database { } async destroy() { - if ( - (await flags.isEnabled(FeatureFlag.SQS)) && - (await this.exists(SQLITE_DESIGN_DOC_ID)) - ) { + if (await this.exists(SQLITE_DESIGN_DOC_ID)) { // delete the design document, then run the cleanup operation const definition = await this.get(SQLITE_DESIGN_DOC_ID) // remove all tables - save the definition then trigger a cleanup diff --git a/packages/pro b/packages/pro index 297fdc937e..14f9c8a925 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865 +Subproject commit 14f9c8a92517bdd08ff29ad39e92cb90d4b2c02f diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 96c01a15b8..d36ac594e7 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -15,12 +15,11 @@ import { getViews, saveView } from "../view/utils" import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { context, events, features } from "@budibase/backend-core" +import { context, events } from "@budibase/backend-core" import { AutoFieldSubType, Database, Datasource, - FeatureFlag, FieldSchema, FieldType, NumberFieldMetadata, @@ -330,9 +329,8 @@ class TableSaveFunctions { importRows: this.importRows, userId: this.userId, }) - if (await features.flags.isEnabled(FeatureFlag.SQS)) { - await sdk.tables.sqs.addTable(table) - } + + await sdk.tables.sqs.addTable(table) return table } @@ -524,9 +522,8 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) { if (rows) { await AttachmentCleanup.tableDelete(table, rows) } - if (await features.flags.isEnabled(FeatureFlag.SQS)) { - await sdk.tables.sqs.removeTable(table) - } + + await sdk.tables.sqs.removeTable(table) } const _TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 7b3d6913cf..efc3c4c93e 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,4 +1,4 @@ -import { context, features } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { getTableParams } from "../../../db/utils" import { breakExternalTableId, @@ -12,7 +12,6 @@ import { TableResponse, TableSourceType, TableViewsResponse, - FeatureFlag, } from "@budibase/types" import datasources from "../datasources" import sdk from "../../../sdk" @@ -39,10 +38,7 @@ export async function processTable(table: Table): Promise { type: "table", sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID, sourceType: TableSourceType.INTERNAL, - } - const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS) - if (sqsEnabled) { - processed.sql = true + sql: true, } return processed } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 6872924f58..a294411e6d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -3,7 +3,6 @@ import { fixAutoColumnSubType, processFormulas } from "./utils" import { cache, context, - features, HTTPError, objectStore, utils, @@ -19,7 +18,6 @@ import { Table, User, ViewV2, - FeatureFlag, } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { @@ -417,44 +415,42 @@ export async function coreOutputProcessing( rows = await processFormulas(table, rows, { dynamic: true }) // remove null properties to match internal API - const isExternal = isExternalTableID(table._id!) - if (isExternal || (await features.flags.isEnabled(FeatureFlag.SQS))) { - for (const row of rows) { - for (const key of Object.keys(row)) { - if (row[key] === null) { - delete row[key] - } else if (row[key] && table.schema[key]?.type === FieldType.LINK) { - for (const link of row[key] || []) { - for (const linkKey of Object.keys(link)) { - if (link[linkKey] === null) { - delete link[linkKey] - } + for (const row of rows) { + for (const key of Object.keys(row)) { + if (row[key] === null) { + delete row[key] + } else if (row[key] && table.schema[key]?.type === FieldType.LINK) { + for (const link of row[key] || []) { + for (const linkKey of Object.keys(link)) { + if (link[linkKey] === null) { + delete link[linkKey] } } } } } - - if (sdk.views.isView(source)) { - const calculationFields = Object.keys( - helpers.views.calculationFields(source) - ) - - // We ensure all calculation fields are returned as numbers. During the - // testing of this feature it was discovered that the COUNT operation - // returns a string for MySQL, MariaDB, and Postgres. But given that all - // calculation fields should be numbers, we blanket make sure of that - // here. - for (const key of calculationFields) { - for (const row of rows) { - if (typeof row[key] === "string") { - row[key] = parseFloat(row[key]) - } - } - } - } } + if (sdk.views.isView(source)) { + const calculationFields = Object.keys( + helpers.views.calculationFields(source) + ) + + // We ensure all calculation fields are returned as numbers. During the + // testing of this feature it was discovered that the COUNT operation + // returns a string for MySQL, MariaDB, and Postgres. But given that all + // calculation fields should be numbers, we blanket make sure of that + // here. + for (const key of calculationFields) { + for (const row of rows) { + if (typeof row[key] === "string") { + row[key] = parseFloat(row[key]) + } + } + } + } + + const isExternal = isExternalTableID(table._id!) if (!isUserMetadataTable(table._id!)) { const protectedColumns = isExternal ? PROTECTED_EXTERNAL_COLUMNS From 9e501b4e659b34e28fc82a160d89138fc995fb6d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 12:52:25 +0200 Subject: [PATCH 002/193] Cleanup SQS feature usages --- .../worker/src/api/controllers/system/environment.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/worker/src/api/controllers/system/environment.ts b/packages/worker/src/api/controllers/system/environment.ts index b6352ea5e7..1d704ada85 100644 --- a/packages/worker/src/api/controllers/system/environment.ts +++ b/packages/worker/src/api/controllers/system/environment.ts @@ -1,6 +1,6 @@ -import { Ctx, MaintenanceType, FeatureFlag } from "@budibase/types" +import { Ctx, MaintenanceType } from "@budibase/types" import env from "../../../environment" -import { env as coreEnv, db as dbCore, features } from "@budibase/backend-core" +import { env as coreEnv, db as dbCore } from "@budibase/backend-core" import nodeFetch from "node-fetch" let sqsAvailable: boolean @@ -29,10 +29,7 @@ async function isSqsAvailable() { } async function isSqsMissing() { - return ( - (await features.flags.isEnabled(FeatureFlag.SQS)) && - !(await isSqsAvailable()) - ) + return !(await isSqsAvailable()) } export const fetch = async (ctx: Ctx) => { From ba5a3d847f535de1bb746466a1ccd8f2b5c98240 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 12:57:04 +0200 Subject: [PATCH 003/193] Cleanup SQS flag from auditlog --- packages/types/src/sdk/db.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index b679d6e182..9797715329 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -12,7 +12,6 @@ import type PouchDB from "pouchdb-find" export enum SearchIndex { ROWS = "rows", - AUDIT = "audit", USER = "user", } From e50f2cc84ea9163ad6923635a3c9acad63a61738 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 13:01:53 +0200 Subject: [PATCH 004/193] Cleanup SQS flag from search --- packages/server/src/sdk/app/rows/search.ts | 5 +- .../src/sdk/app/rows/search/internal/index.ts | 1 - .../sdk/app/rows/search/internal/lucene.ts | 79 ------------------- 3 files changed, 1 insertion(+), 84 deletions(-) delete mode 100644 packages/server/src/sdk/app/rows/search/internal/lucene.ts diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f80c1c1f8a..6b9d068373 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -169,12 +169,9 @@ export async function search( if (isExternalTable) { span?.addTags({ searchType: "external" }) result = await external.search(options, source) - } else if (await features.flags.isEnabled(FeatureFlag.SQS)) { + } else { span?.addTags({ searchType: "sqs" }) result = await internal.sqs.search(options, source) - } else { - span?.addTags({ searchType: "lucene" }) - result = await internal.lucene.search(options, source) } span.addTags({ diff --git a/packages/server/src/sdk/app/rows/search/internal/index.ts b/packages/server/src/sdk/app/rows/search/internal/index.ts index f3db9169f4..58d1bd9c96 100644 --- a/packages/server/src/sdk/app/rows/search/internal/index.ts +++ b/packages/server/src/sdk/app/rows/search/internal/index.ts @@ -1,3 +1,2 @@ export * as sqs from "./sqs" -export * as lucene from "./lucene" export * from "./internal" diff --git a/packages/server/src/sdk/app/rows/search/internal/lucene.ts b/packages/server/src/sdk/app/rows/search/internal/lucene.ts deleted file mode 100644 index 953fb90c1f..0000000000 --- a/packages/server/src/sdk/app/rows/search/internal/lucene.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" -import { fullSearch, paginatedSearch } from "../utils" -import { InternalTables } from "../../../../../db/utils" -import { - Row, - RowSearchParams, - SearchResponse, - SortType, - Table, - User, - ViewV2, -} from "@budibase/types" -import { getGlobalUsersFromMetadata } from "../../../../../utilities/global" -import { outputProcessing } from "../../../../../utilities/rowProcessor" -import pick from "lodash/pick" -import sdk from "../../../../" - -export async function search( - options: RowSearchParams, - source: Table | ViewV2 -): Promise> { - let table: Table - if (sdk.views.isView(source)) { - table = await sdk.views.getTable(source.id) - } else { - table = source - } - - const { paginate, query } = options - - const params: RowSearchParams = { - tableId: options.tableId, - viewId: options.viewId, - sort: options.sort, - sortOrder: options.sortOrder, - sortType: options.sortType, - limit: options.limit, - bookmark: options.bookmark, - version: options.version, - disableEscaping: options.disableEscaping, - query: {}, - } - - if (params.sort && !params.sortType) { - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = - sortField.type === "number" ? SortType.NUMBER : SortType.STRING - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (table._id === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) - } - - const visibleFields = - options.fields || - Object.keys(source.schema || {}).filter( - key => source.schema?.[key].visible !== false - ) - const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS] - response.rows = response.rows.map((r: any) => pick(r, allowedFields)) - - response.rows = await outputProcessing(source, response.rows, { - squash: true, - }) - } - - return response -} From 5b1ed8195afc851327a06b4bc1d709f2c927f031 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 13:03:22 +0200 Subject: [PATCH 005/193] Cleanup SQS flag from view search --- packages/server/src/sdk/app/rows/search.ts | 52 ++++------------------ 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 6b9d068373..a2b612dbb9 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,11 +1,7 @@ import { EmptyFilterOption, - FeatureFlag, - LegacyFilter, - LogicalOperator, Row, RowSearchParams, - SearchFilterKey, SearchResponse, SortOrder, Table, @@ -18,7 +14,6 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../index" import { checkFilters, searchInputMapping } from "./search/utils" -import { db, features } from "@budibase/backend-core" import tracer from "dd-trace" import { getQueryableFields, removeInvalidFilters } from "./queryUtils" import { enrichSearchContext } from "../../../api/controllers/row/utils" @@ -102,45 +97,14 @@ export async function search( viewQuery = checkFilters(table, viewQuery) delete viewQuery?.onEmptyFilter - const sqsEnabled = await features.flags.isEnabled(FeatureFlag.SQS) - const supportsLogicalOperators = - isExternalTableID(view.tableId) || sqsEnabled - - if (!supportsLogicalOperators) { - // In the unlikely event that a Grouped Filter is in a non-SQS environment - // It needs to be ignored entirely - let queryFilters: LegacyFilter[] = Array.isArray(view.query) - ? view.query - : [] - - delete options.query.onEmptyFilter - - // Extract existing fields - const existingFields = - queryFilters - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] - - // Carry over filters for unused fields - Object.keys(options.query).forEach(key => { - const operator = key as Exclude - Object.keys(options.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { - viewQuery[operator]![field] = options.query[operator]![field] - } - }) - }) - options.query = viewQuery - } else { - const conditions = viewQuery ? [viewQuery] : [] - options.query = { - $and: { - conditions: [...conditions, options.query], - }, - } - if (viewQuery.onEmptyFilter) { - options.query.onEmptyFilter = viewQuery.onEmptyFilter - } + const conditions = viewQuery ? [viewQuery] : [] + options.query = { + $and: { + conditions: [...conditions, options.query], + }, + } + if (viewQuery.onEmptyFilter) { + options.query.onEmptyFilter = viewQuery.onEmptyFilter } } From 655cd353e7aa84938718db9309b26ffe807c5c47 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 13:06:53 +0200 Subject: [PATCH 006/193] Remove lucene from row tests --- .../server/src/api/routes/tests/row.spec.ts | 194 +----------------- 1 file changed, 1 insertion(+), 193 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 31067ab40f..bd8fbb8c79 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -18,17 +18,14 @@ import { } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { - AttachmentFieldMetadata, AutoFieldSubType, Datasource, - DateFieldMetadata, DeleteRow, FieldSchema, FieldType, BBReferenceFieldSubType, FormulaType, INTERNAL_TABLE_SOURCE_ID, - NumberFieldMetadata, QuotaUsageType, RelationshipType, Row, @@ -76,8 +73,7 @@ async function waitForEvent( } describe.each([ - ["lucene", undefined], - ["sqs", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], @@ -85,8 +81,6 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined - const isLucene = providerType === "lucene" - const isSqs = providerType === "sqs" const isMSSQL = providerType === DatabaseName.SQL_SERVER const isOracle = providerType === DatabaseName.ORACLE const config = setup.getConfig() @@ -94,15 +88,11 @@ describe.each([ let table: Table let datasource: Datasource | undefined let client: Knex | undefined - let envCleanup: (() => void) | undefined beforeAll(async () => { await features.testutils.withFeatureFlags("*", { SQS: true }, () => config.init() ) - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) if (dsProvider) { const rawDatasource = await dsProvider @@ -115,9 +105,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) function saveTableRequest( @@ -367,185 +354,6 @@ describe.each([ expect(ids).toEqual(expect.arrayContaining(sequence)) }) - isLucene && - it("row values are coerced", async () => { - const str: FieldSchema = { - type: FieldType.STRING, - name: "str", - constraints: { type: "string", presence: false }, - } - const singleAttachment: FieldSchema = { - type: FieldType.ATTACHMENT_SINGLE, - name: "single attachment", - constraints: { presence: false }, - } - const attachmentList: AttachmentFieldMetadata = { - type: FieldType.ATTACHMENTS, - name: "attachments", - constraints: { type: "array", presence: false }, - } - const signature: FieldSchema = { - type: FieldType.SIGNATURE_SINGLE, - name: "signature", - constraints: { presence: false }, - } - const bool: FieldSchema = { - type: FieldType.BOOLEAN, - name: "boolean", - constraints: { type: "boolean", presence: false }, - } - const number: NumberFieldMetadata = { - type: FieldType.NUMBER, - name: "str", - constraints: { type: "number", presence: false }, - } - const datetime: DateFieldMetadata = { - type: FieldType.DATETIME, - name: "datetime", - constraints: { - type: "string", - presence: false, - datetime: { earliest: "", latest: "" }, - }, - } - const arrayField: FieldSchema = { - type: FieldType.ARRAY, - constraints: { - type: JsonFieldSubType.ARRAY, - presence: false, - inclusion: ["One", "Two", "Three"], - }, - name: "Sample Tags", - sortable: false, - } - const optsField: FieldSchema = { - name: "Sample Opts", - type: FieldType.OPTIONS, - constraints: { - type: "string", - presence: false, - inclusion: ["Alpha", "Beta", "Gamma"], - }, - } - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: str, - stringUndefined: str, - stringNull: str, - stringString: str, - numberEmptyString: number, - numberNull: number, - numberUndefined: number, - numberString: number, - numberNumber: number, - datetimeEmptyString: datetime, - datetimeNull: datetime, - datetimeUndefined: datetime, - datetimeString: datetime, - datetimeDate: datetime, - boolNull: bool, - boolEmpty: bool, - boolUndefined: bool, - boolString: bool, - boolBool: bool, - singleAttachmentNull: singleAttachment, - singleAttachmentUndefined: singleAttachment, - attachmentListNull: attachmentList, - attachmentListUndefined: attachmentList, - attachmentListEmpty: attachmentList, - attachmentListEmptyArrayStr: attachmentList, - signatureNull: signature, - signatureUndefined: signature, - arrayFieldEmptyArrayStr: arrayField, - arrayFieldArrayStrKnown: arrayField, - arrayFieldNull: arrayField, - arrayFieldUndefined: arrayField, - optsFieldEmptyStr: optsField, - optsFieldUndefined: optsField, - optsFieldNull: optsField, - optsFieldStrKnown: optsField, - }, - }) - ) - - const datetimeStr = "1984-04-20T00:00:00.000Z" - - const row = await config.api.row.save(table._id!, { - name: "Test Row", - stringUndefined: undefined, - stringNull: null, - stringString: "i am a string", - numberEmptyString: "", - numberNull: null, - numberUndefined: undefined, - numberString: "123", - numberNumber: 123, - datetimeEmptyString: "", - datetimeNull: null, - datetimeUndefined: undefined, - datetimeString: datetimeStr, - datetimeDate: new Date(datetimeStr), - boolNull: null, - boolEmpty: "", - boolUndefined: undefined, - boolString: "true", - boolBool: true, - tableId: table._id, - singleAttachmentNull: null, - singleAttachmentUndefined: undefined, - attachmentListNull: null, - attachmentListUndefined: undefined, - attachmentListEmpty: "", - attachmentListEmptyArrayStr: "[]", - signatureNull: null, - signatureUndefined: undefined, - arrayFieldEmptyArrayStr: "[]", - arrayFieldUndefined: undefined, - arrayFieldNull: null, - arrayFieldArrayStrKnown: "['One']", - optsFieldEmptyStr: "", - optsFieldUndefined: undefined, - optsFieldNull: null, - optsFieldStrKnown: "Alpha", - }) - - expect(row.stringUndefined).toBe(undefined) - expect(row.stringNull).toBe(null) - expect(row.stringString).toBe("i am a string") - expect(row.numberEmptyString).toBe(null) - expect(row.numberNull).toBe(null) - expect(row.numberUndefined).toBe(undefined) - expect(row.numberString).toBe(123) - expect(row.numberNumber).toBe(123) - expect(row.datetimeEmptyString).toBe(null) - expect(row.datetimeNull).toBe(null) - expect(row.datetimeUndefined).toBe(undefined) - expect(row.datetimeString).toBe(new Date(datetimeStr).toISOString()) - expect(row.datetimeDate).toBe(new Date(datetimeStr).toISOString()) - expect(row.boolNull).toBe(null) - expect(row.boolEmpty).toBe(null) - expect(row.boolUndefined).toBe(undefined) - expect(row.boolString).toBe(true) - expect(row.boolBool).toBe(true) - expect(row.singleAttachmentNull).toEqual(null) - expect(row.singleAttachmentUndefined).toBe(undefined) - expect(row.attachmentListNull).toEqual([]) - expect(row.attachmentListUndefined).toBe(undefined) - expect(row.attachmentListEmpty).toEqual([]) - expect(row.attachmentListEmptyArrayStr).toEqual([]) - expect(row.signatureNull).toEqual(null) - expect(row.signatureUndefined).toBe(undefined) - expect(row.arrayFieldEmptyArrayStr).toEqual([]) - expect(row.arrayFieldNull).toEqual([]) - expect(row.arrayFieldUndefined).toEqual(undefined) - expect(row.optsFieldEmptyStr).toEqual(null) - expect(row.optsFieldUndefined).toEqual(undefined) - expect(row.optsFieldNull).toEqual(null) - expect(row.arrayFieldArrayStrKnown).toEqual(["One"]) - expect(row.optsFieldStrKnown).toEqual("Alpha") - }) - isInternal && it("doesn't allow creating in user table", async () => { const response = await config.api.row.save( From 08a9488194cc4e5a84bbbd6e32f67953ba609d73 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 13:50:03 +0200 Subject: [PATCH 007/193] Remove lucene from search tests --- .../src/api/routes/tests/search.spec.ts | 914 +++++++++--------- packages/shared-core/src/filters.ts | 7 +- 2 files changed, 442 insertions(+), 479 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index e0143c5938..11594f5d82 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -49,7 +49,6 @@ import { cloneDeep } from "lodash/fp" describe.each([ ["in-memory", undefined], - ["lucene", undefined], ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], @@ -57,14 +56,11 @@ describe.each([ [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { - const isSqs = name === "sqs" - const isLucene = name === "lucene" const isInMemory = name === "in-memory" - const isInternal = isSqs || isLucene || isInMemory - const isSql = !isInMemory && !isLucene + const isInternal = !dsProvider + const isSql = !isInMemory const config = setup.getConfig() - let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let client: Knex | undefined let tableOrViewId: string @@ -98,9 +94,6 @@ describe.each([ await features.testutils.withFeatureFlags("*", { SQS: true }, () => config.init() ) - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { @@ -124,9 +117,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) async function createTable(schema: TableSchema) { @@ -176,11 +166,6 @@ describe.each([ ])("from %s", (sourceType, createTableOrView) => { const isView = sourceType === "view" - if (isView && isLucene) { - // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests - return - } - class SearchAssertion { constructor(private readonly query: SearchRowRequest) {} @@ -553,19 +538,18 @@ describe.each([ ]) }) - !isLucene && - it("should return all rows matching the session user firstname when logical operator used", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { name: "{{ [user].firstName }}" } }], - }, - }).toContainExactly([ - { - name: config.getUser().firstName, - appointment: future.toISOString(), - }, - ]) - }) + it("should return all rows matching the session user firstname when logical operator used", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { name: "{{ [user].firstName }}" } }], + }, + }).toContainExactly([ + { + name: config.getUser().firstName, + appointment: future.toISOString(), + }, + ]) + }) it("should parse the date binding and return all rows after the resolved value", async () => { await tk.withFreeze(serverTime, async () => { @@ -988,21 +972,19 @@ describe.each([ }).toFindNothing() }) - !isLucene && - it("ignores low if it's an empty object", async () => { - await expectQuery({ - // @ts-ignore - range: { name: { low: {}, high: "z" } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) + it("ignores low if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: {}, high: "z" } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - !isLucene && - it("ignores high if it's an empty object", async () => { - await expectQuery({ - // @ts-ignore - range: { name: { low: "a", high: {} } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }]) - }) + it("ignores high if it's an empty object", async () => { + await expectQuery({ + // @ts-ignore + range: { name: { low: "a", high: {} } }, + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("empty", () => { @@ -1156,31 +1138,23 @@ describe.each([ await expectQuery({ oneOf: { age: [2] } }).toFindNothing() }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can convert from a string", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - age: "1", - }, - }).toContainExactly([{ age: 1 }]) - }) + it("can convert from a string", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1", + }, + }).toContainExactly([{ age: 1 }]) + }) - // I couldn't find a way to make this work in Lucene and given that - // we're getting rid of Lucene soon I wasn't inclined to spend time on - // it. - !isLucene && - it("can find multiple values for same column", async () => { - await expectQuery({ - oneOf: { - // @ts-ignore - age: "1,10", - }, - }).toContainExactly([{ age: 1 }, { age: 10 }]) - }) + it("can find multiple values for same column", async () => { + await expectQuery({ + oneOf: { + // @ts-ignore + age: "1,10", + }, + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) }) describe("range", () => { @@ -1760,47 +1734,43 @@ describe.each([ }) }) - // Range searches against bigints don't seem to work at all in Lucene, and I - // couldn't figure out why. Given that we're replacing Lucene with SQS, - // we've decided not to spend time on it. - !isLucene && - describe("range", () => { - it("successfully finds a row", async () => { - await expectQuery({ - range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }]) - }) - - it("successfully finds multiple rows", async () => { - await expectQuery({ - range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) - }) - - it("successfully finds a row with a high bound", async () => { - await expectQuery({ - range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("successfully finds no rows", async () => { - await expectQuery({ - range: { num: { low: "5", high: "5" } }, - }).toFindNothing() - }) - - it("can search using just a low value", async () => { - await expectQuery({ - range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) - }) - - it("can search using just a high value", async () => { - await expectQuery({ - range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) - }) + describe("range", () => { + it("successfully finds a row", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }]) }) + + it("successfully finds multiple rows", async () => { + await expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + + it("successfully finds a row with a high bound", async () => { + await expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("successfully finds no rows", async () => { + await expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing() + }) + + it("can search using just a low value", async () => { + await expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) + + it("can search using just a high value", async () => { + await expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) + }) }) isInternal && @@ -1897,94 +1867,93 @@ describe.each([ }).toFindNothing() }) - isSqs && - it("can search using just a low value", async () => { - await expectQuery({ - range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }]) - }) + it("can search using just a low value", async () => { + await expectQuery({ + range: { auto: { low: 9 } }, + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) - isSqs && - it("can search using just a high value", async () => { - await expectQuery({ - range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }]) - }) + it("can search using just a high value", async () => { + await expectQuery({ + range: { auto: { high: 2 } }, + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) }) - isSqs && - describe("sort", () => { - it("sorts ascending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([ - { auto: 1 }, - { auto: 2 }, - { auto: 3 }, - { auto: 4 }, - { auto: 5 }, - { auto: 6 }, - { auto: 7 }, - { auto: 8 }, - { auto: 9 }, - { auto: 10 }, - ]) - }) - - it("sorts descending", async () => { - await expectSearch({ - query: {}, - sort: "auto", - sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([ - { auto: 10 }, - { auto: 9 }, - { auto: 8 }, - { auto: 7 }, - { auto: 6 }, - { auto: 5 }, - { auto: 4 }, - { auto: 3 }, - { auto: 2 }, - { auto: 1 }, - ]) - }) - - // This is important for pagination. The order of results must always - // be stable or pagination will break. We don't want the user to need - // to specify an order for pagination to work. - it("is stable without a sort specified", async () => { - let { rows: fullRowList } = await config.api.row.search( - tableOrViewId, - { - tableId: tableOrViewId, - query: {}, - } - ) - - // repeat the search many times to check the first row is always the same - let bookmark: string | number | undefined, - hasNextPage: boolean | undefined = true, - rowCount: number = 0 - do { - const response = await config.api.row.search(tableOrViewId, { - tableId: tableOrViewId, - limit: 1, - paginate: true, - query: {}, - bookmark, - }) - bookmark = response.bookmark - hasNextPage = response.hasNextPage - expect(response.rows.length).toEqual(1) - const foundRow = response.rows[0] - expect(foundRow).toEqual(fullRowList[rowCount++]) - } while (hasNextPage) - }) + describe("sort", () => { + it("sorts ascending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.ASCENDING, + sortType: SortType.NUMBER, + }).toMatchExactly([ + { auto: 1 }, + { auto: 2 }, + { auto: 3 }, + { auto: 4 }, + { auto: 5 }, + { auto: 6 }, + { auto: 7 }, + { auto: 8 }, + { auto: 9 }, + { auto: 10 }, + ]) }) + it("sorts descending", async () => { + await expectSearch({ + query: {}, + sort: "auto", + sortOrder: SortOrder.DESCENDING, + sortType: SortType.NUMBER, + }).toMatchExactly([ + { auto: 10 }, + { auto: 9 }, + { auto: 8 }, + { auto: 7 }, + { auto: 6 }, + { auto: 5 }, + { auto: 4 }, + { auto: 3 }, + { auto: 2 }, + { auto: 1 }, + ]) + }) + + // This is important for pagination. The order of results must always + // be stable or pagination will break. We don't want the user to need + // to specify an order for pagination to work. + it("is stable without a sort specified", async () => { + let { rows: fullRowList } = await config.api.row.search( + tableOrViewId, + { + tableId: tableOrViewId, + query: {}, + } + ) + + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { + const response = await config.api.row.search(tableOrViewId, { + tableId: tableOrViewId, + limit: 1, + paginate: true, + query: {}, + bookmark, + }) + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) + }) + }) + describe("pagination", () => { it("should paginate through all rows", async () => { // @ts-ignore @@ -2273,11 +2242,9 @@ describe.each([ }) }) - // This will never work for Lucene. - !isLucene && - // It also can't work for in-memory searching because the related table name - // isn't available. - !isInMemory && + // It also can't work for in-memory searching because the related table name + // isn't available. + !isInMemory && describe.each([ RelationshipType.ONE_TO_MANY, RelationshipType.MANY_TO_ONE, @@ -2728,42 +2695,40 @@ describe.each([ }) }) - // lucene can't count the total rows - !isLucene && - describe("row counting", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - name: { - name: "name", - type: FieldType.STRING, - }, - }) - await createRows([{ name: "a" }, { name: "b" }]) - }) - - it("should be able to count rows when option set", async () => { - await expectSearch({ - countRows: true, - query: { - notEmpty: { - name: true, - }, - }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) }) - }) - - it("shouldn't count rows when option is not set", async () => { - await expectSearch({ - countRows: false, - query: { - notEmpty: { - name: true, - }, - }, - }).toNotHaveProperty(["totalRows"]) + describe("row counting", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + name: { + name: "name", + type: FieldType.STRING, + }, }) + await createRows([{ name: "a" }, { name: "b" }]) }) + it("should be able to count rows when option set", async () => { + await expectSearch({ + countRows: true, + query: { + notEmpty: { + name: true, + }, + }, + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) + + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ + countRows: false, + query: { + notEmpty: { + name: true, + }, + }, + }).toNotHaveProperty(["totalRows"]) + }) + }) + describe("Invalid column definitions", () => { beforeAll(async () => { // need to create an invalid table - means ignoring typescript @@ -2946,9 +2911,7 @@ describe.each([ }) }) - // This was never actually supported in Lucene but SQS does support it, so may - // as well have a test for it. - ;(isSqs || isInMemory) && + isInternal && describe("space at start of column name", () => { beforeAll(async () => { tableOrViewId = await createTableOrView({ @@ -2981,7 +2944,7 @@ describe.each([ }) }) - isSqs && + isInternal && !isView && describe("duplicate columns", () => { beforeAll(async () => { @@ -3143,291 +3106,286 @@ describe.each([ }) }) - !isLucene && - describe("$and", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) + describe("$and", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) - }) + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([{ age: 10, name: "Jack" }]) - }) + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([{ age: 10, name: "Jack" }]) + }) - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $and: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Ja" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) - }) + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $and: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Ja" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $and: { - conditions: [ - { - $and: { - conditions: [ - { - range: { age: { low: 1, high: 10 } }, - }, - { string: { name: "Ja" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([{ age: 1, name: "Jane" }]) - }) - - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $and: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toFindNothing() - }) - - !isInMemory && - it("validates conditions that are not objects", async () => { - await expect( - expectQuery({ + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { $and: { conditions: [ - { equal: { age: 10 } }, - "invalidCondition" as any, - ], - }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1]" must be of type object' - ) - }) - - !isInMemory && - it("validates $and without conditions", async () => { - await expect( - expectQuery({ - $and: { - conditions: [ - { equal: { age: 10 } }, { - $and: { - conditions: undefined as any, - }, + range: { age: { low: 1, high: 10 } }, }, + { string: { name: "Ja" } }, ], }, - }).toFindNothing() - ).rejects.toThrow( - 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' - ) - }) + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) - // onEmptyFilter cannot be sent to view searches - !isView && - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - $and: { - conditions: [{ equal: { name: "" } }], - }, + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $and: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toFindNothing() + }) + + !isInMemory && + it("validates conditions that are not objects", async () => { + await expect( + expectQuery({ + $and: { + conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], }, }).toFindNothing() - }) + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1]" must be of type object' + ) + }) - it("returns all rows when onEmptyFilter set to all", async () => { + !isInMemory && + it("validates $and without conditions", async () => { + await expect( + expectQuery({ + $and: { + conditions: [ + { equal: { age: 10 } }, + { + $and: { + conditions: undefined as any, + }, + }, + ], + }, + }).toFindNothing() + ).rejects.toThrow( + 'Invalid body - "query.$and.conditions[1].$and.conditions" is required' + ) + }) + + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { await expectSearch({ query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, $and: { conditions: [{ equal: { name: "" } }], }, }, - }).toHaveLength(4) - }) - }) - - !isLucene && - describe("$or", () => { - beforeAll(async () => { - tableOrViewId = await createTableOrView({ - age: { name: "age", type: FieldType.NUMBER }, - name: { name: "name", type: FieldType.STRING }, - }) - await createRows([ - { age: 1, name: "Jane" }, - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds a row for one level condition", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds a row for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], - }, - }).toContainExactly([ - { age: 10, name: "Jack" }, - { age: 7, name: "Hanna" }, - ]) - }) - - it("successfully finds multiple rows for one level with multiple conditions", async () => { - await expectQuery({ - $or: { - conditions: [ - { range: { age: { low: 1, high: 9 } } }, - { string: { name: "Jan" } }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("successfully finds rows for nested filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $or: { - conditions: [ - { - range: { age: { low: 1, high: 7 } }, - }, - { string: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 7, name: "Hanna" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("returns nothing when filtering out all data", async () => { - await expectQuery({ - $or: { - conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], - }, }).toFindNothing() }) - it("can nest $and under $or filters", async () => { - await expectQuery({ - $or: { - conditions: [ - { - $and: { - conditions: [ - { - range: { age: { low: 1, high: 8 } }, - }, - { equal: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], - }, - }).toContainExactly([ - { age: 1, name: "Jane" }, - { age: 8, name: "Jan" }, - ]) - }) - - it("can nest $or under $and filters", async () => { - await expectQuery({ + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, $and: { - conditions: [ - { - $or: { - conditions: [ - { - range: { age: { low: 1, high: 8 } }, - }, - { equal: { name: "Jan" } }, - ], - }, - equal: { name: "Jane" }, - }, - ], + conditions: [{ equal: { name: "" } }], }, - }).toContainExactly([{ age: 1, name: "Jane" }]) + }, + }).toHaveLength(4) + }) + }) + + describe("$or", () => { + beforeAll(async () => { + tableOrViewId = await createTableOrView({ + age: { name: "age", type: FieldType.NUMBER }, + name: { name: "name", type: FieldType.STRING }, }) + await createRows([ + { age: 1, name: "Jane" }, + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - // onEmptyFilter cannot be sent to view searches - !isView && - it("returns no rows when onEmptyFilter set to none", async () => { - await expectSearch({ - query: { - onEmptyFilter: EmptyFilterOption.RETURN_NONE, + it("successfully finds a row for one level condition", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds a row for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }], + }, + }).toContainExactly([ + { age: 10, name: "Jack" }, + { age: 7, name: "Hanna" }, + ]) + }) + + it("successfully finds multiple rows for one level with multiple conditions", async () => { + await expectQuery({ + $or: { + conditions: [ + { range: { age: { low: 1, high: 9 } } }, + { string: { name: "Jan" } }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("successfully finds rows for nested filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { $or: { - conditions: [{ equal: { name: "" } }], + conditions: [ + { + range: { age: { low: 1, high: 7 } }, + }, + { string: { name: "Jan" } }, + ], }, + equal: { name: "Jane" }, }, - }).toFindNothing() - }) + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 7, name: "Hanna" }, + { age: 8, name: "Jan" }, + ]) + }) - it("returns all rows when onEmptyFilter set to all", async () => { + it("returns nothing when filtering out all data", async () => { + await expectQuery({ + $or: { + conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }], + }, + }).toFindNothing() + }) + + it("can nest $and under $or filters", async () => { + await expectQuery({ + $or: { + conditions: [ + { + $and: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([ + { age: 1, name: "Jane" }, + { age: 8, name: "Jan" }, + ]) + }) + + it("can nest $or under $and filters", async () => { + await expectQuery({ + $and: { + conditions: [ + { + $or: { + conditions: [ + { + range: { age: { low: 1, high: 8 } }, + }, + { equal: { name: "Jan" } }, + ], + }, + equal: { name: "Jane" }, + }, + ], + }, + }).toContainExactly([{ age: 1, name: "Jane" }]) + }) + + // onEmptyFilter cannot be sent to view searches + !isView && + it("returns no rows when onEmptyFilter set to none", async () => { await expectSearch({ query: { - onEmptyFilter: EmptyFilterOption.RETURN_ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, $or: { conditions: [{ equal: { name: "" } }], }, }, - }).toHaveLength(4) + }).toFindNothing() }) + + it("returns all rows when onEmptyFilter set to all", async () => { + await expectSearch({ + query: { + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + $or: { + conditions: [{ equal: { name: "" } }], + }, + }, + }).toHaveLength(4) }) + }) isSql && describe("max related columns", () => { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 40c70a8a23..fa58f938bd 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -645,7 +645,12 @@ export function search>( ): SearchResponse { let result = runQuery(docs, query.query) if (query.sort) { - result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) + result = sort( + result, + query.sort, + query.sortOrder || SortOrder.ASCENDING, + query.sortType + ) } const totalRows = result.length if (query.limit) { From 718a2a4d87b0a167ee513a812b845246460543a1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:14:35 +0200 Subject: [PATCH 008/193] Remove lucene from viewV2 tests --- .../src/api/routes/tests/viewV2.spec.ts | 1538 ++++++++--------- 1 file changed, 733 insertions(+), 805 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index dfd4f50bd1..fe44b495e3 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -35,8 +35,7 @@ import { quotas } from "@budibase/pro" import { db, roles, features } from "@budibase/backend-core" describe.each([ - ["lucene", undefined], - ["sqs", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], @@ -44,13 +43,10 @@ describe.each([ [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() - const isSqs = name === "sqs" - const isLucene = name === "lucene" - const isInternal = isSqs || isLucene + const isInternal = name === "internal" let table: Table let datasource: Datasource - let envCleanup: (() => void) | undefined function saveTableRequest( ...overrides: Partial>[] @@ -97,14 +93,6 @@ describe.each([ } beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => - config.init() - ) - - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) - if (dsProvider) { datasource = await config.createDatasource({ datasource: await dsProvider, @@ -115,9 +103,6 @@ describe.each([ afterAll(async () => { setup.afterAll() - if (envCleanup) { - envCleanup() - } }) beforeEach(() => { @@ -738,41 +723,40 @@ describe.each([ }) }) - !isLucene && - it("does not get confused when a calculation field shadows a basic one", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - age: { - name: "age", - type: FieldType.NUMBER, - }, - }, - }) - ) - - await config.api.row.bulkImport(table._id!, { - rows: [{ age: 1 }, { age: 2 }, { age: 3 }], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, + it("does not get confused when a calculation field shadows a basic one", async () => { + const table = await config.api.table.save( + saveTableRequest({ schema: { age: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", + name: "age", + type: FieldType.NUMBER, }, }, }) + ) - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].age).toEqual(6) + await config.api.row.bulkImport(table._id!, { + rows: [{ age: 1 }, { age: 2 }, { age: 3 }], }) + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + age: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].age).toEqual(6) + }) + // We don't allow the creation of tables with most JsonTypes when using // external datasources. isInternal && @@ -1153,205 +1137,204 @@ describe.each([ ) }) - !isLucene && - describe("calculation views", () => { - let table: Table - let view: ViewV2 + describe("calculation views", () => { + let table: Table + let view: ViewV2 - beforeEach(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - country: { - name: "country", - type: FieldType.STRING, - }, - age: { - name: "age", - type: FieldType.NUMBER, + beforeEach(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, }, }, - }) - ) - - view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { country: { - visible: true, + name: "country", + type: FieldType.STRING, }, age: { - visible: true, - calculationType: CalculationType.SUM, - field: "age", + name: "age", + type: FieldType.NUMBER, }, }, }) + ) - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "Steve", - age: 30, - country: "UK", - }, - { - name: "Jane", - age: 31, - country: "UK", - }, - { - name: "Ruari", - age: 32, - country: "USA", - }, - { - name: "Alice", - age: 33, - country: "USA", - }, - ], - }) + view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + country: { + visible: true, + }, + age: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, }) - it("returns the expected rows prior to modification", async () => { - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - }, - { - country: "UK", - age: 61, - }, - ]) - ) - }) - - it("can remove a group by field", async () => { - delete view.schema!.country - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows).toEqual( - expect.arrayContaining([ - { - age: 126, - }, - ]) - ) - }) - - it("can remove a calculation field", async () => { - delete view.schema!.age - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(4) - - // Because the removal of the calculation field actually makes this - // no longer a calculation view, these rows will now have _id and - // _rev fields. - expect(rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ country: "UK" }), - expect.objectContaining({ country: "UK" }), - expect.objectContaining({ country: "USA" }), - expect.objectContaining({ country: "USA" }), - ]) - ) - }) - - it("can add a new group by field", async () => { - view.schema!.name = { visible: true } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(4) - expect(rows).toEqual( - expect.arrayContaining([ - { - name: "Steve", - age: 30, - country: "UK", - }, - { - name: "Jane", - age: 31, - country: "UK", - }, - { - name: "Ruari", - age: 32, - country: "USA", - }, - { - name: "Alice", - age: 33, - country: "USA", - }, - ]) - ) - }) - - it("can add a new group by field that is invisible, even if required on the table", async () => { - view.schema!.name = { visible: false } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - }, - { - country: "UK", - age: 61, - }, - ]) - ) - }) - - it("can add a new calculation field", async () => { - view.schema!.count = { - visible: true, - calculationType: CalculationType.COUNT, - } - await config.api.viewV2.update(view) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(2) - expect(rows).toEqual( - expect.arrayContaining([ - { - country: "USA", - age: 65, - count: 2, - }, - { - country: "UK", - age: 61, - count: 2, - }, - ]) - ) + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Steve", + age: 30, + country: "UK", + }, + { + name: "Jane", + age: 31, + country: "UK", + }, + { + name: "Ruari", + age: 32, + country: "USA", + }, + { + name: "Alice", + age: 33, + country: "USA", + }, + ], }) }) + + it("returns the expected rows prior to modification", async () => { + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + }, + { + country: "UK", + age: 61, + }, + ]) + ) + }) + + it("can remove a group by field", async () => { + delete view.schema!.country + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows).toEqual( + expect.arrayContaining([ + { + age: 126, + }, + ]) + ) + }) + + it("can remove a calculation field", async () => { + delete view.schema!.age + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(4) + + // Because the removal of the calculation field actually makes this + // no longer a calculation view, these rows will now have _id and + // _rev fields. + expect(rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ country: "UK" }), + expect.objectContaining({ country: "UK" }), + expect.objectContaining({ country: "USA" }), + expect.objectContaining({ country: "USA" }), + ]) + ) + }) + + it("can add a new group by field", async () => { + view.schema!.name = { visible: true } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(4) + expect(rows).toEqual( + expect.arrayContaining([ + { + name: "Steve", + age: 30, + country: "UK", + }, + { + name: "Jane", + age: 31, + country: "UK", + }, + { + name: "Ruari", + age: 32, + country: "USA", + }, + { + name: "Alice", + age: 33, + country: "USA", + }, + ]) + ) + }) + + it("can add a new group by field that is invisible, even if required on the table", async () => { + view.schema!.name = { visible: false } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + }, + { + country: "UK", + age: 61, + }, + ]) + ) + }) + + it("can add a new calculation field", async () => { + view.schema!.count = { + visible: true, + calculationType: CalculationType.COUNT, + } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + count: 2, + }, + { + country: "UK", + age: 61, + count: 2, + }, + ]) + ) + }) + }) }) describe("delete", () => { @@ -2484,9 +2467,6 @@ describe.each([ hasNextPage: false, totalRows: 10, } - if (isLucene) { - expectation.bookmark = expect.anything() - } expect(page3).toEqual(expectation) }) @@ -2758,92 +2738,10 @@ describe.each([ ) }) - isLucene && - it.each([true, false])( - "in lucene, cannot override a view filter", - async allOr => { - await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - const two = await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - query: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", - }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: two._id }), - ]) - } - ) - - !isLucene && - it.each([true, false])( - "can filter a view without a view filter", - async allOr => { - const one = await config.api.row.save(table._id!, { - one: "foo", - two: "bar", - }) - await config.api.row.save(table._id!, { - one: "foo2", - two: "bar2", - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - schema: { - id: { visible: true }, - one: { visible: false }, - two: { visible: true }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: { - allOr, - equal: { - two: "bar", - }, - }, - }) - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual([ - expect.objectContaining({ _id: one._id }), - ]) - } - ) - - !isLucene && - it.each([true, false])("cannot bypass a view filter", async allOr => { - await config.api.row.save(table._id!, { + it.each([true, false])( + "can filter a view without a view filter", + async allOr => { + const one = await config.api.row.save(table._id!, { one: "foo", two: "bar", }) @@ -2855,13 +2753,6 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), - query: [ - { - operator: BasicOperator.EQUAL, - field: "two", - value: "bar2", - }, - ], schema: { id: { visible: true }, one: { visible: false }, @@ -2877,8 +2768,50 @@ describe.each([ }, }, }) - expect(response.rows).toHaveLength(0) + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual([ + expect.objectContaining({ _id: one._id }), + ]) + } + ) + + it.each([true, false])("cannot bypass a view filter", async allOr => { + await config.api.row.save(table._id!, { + one: "foo", + two: "bar", }) + await config.api.row.save(table._id!, { + one: "foo2", + two: "bar2", + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + query: [ + { + operator: BasicOperator.EQUAL, + field: "two", + value: "bar2", + }, + ], + schema: { + id: { visible: true }, + one: { visible: false }, + two: { visible: true }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: { + allOr, + equal: { + two: "bar", + }, + }, + }) + expect(response.rows).toHaveLength(0) + }) describe("foreign relationship columns", () => { let envCleanup: () => void @@ -3041,500 +2974,46 @@ describe.each([ }) }) - !isLucene && - describe("calculations", () => { - let table: Table - let rows: Row[] + describe("calculations", () => { + let table: Table + let rows: Row[] - beforeAll(async () => { - table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - rows = await Promise.all( - Array.from({ length: 10 }, () => - config.api.row.save(table._id!, { - quantity: generator.natural({ min: 1, max: 10 }), - price: generator.natural({ min: 1, max: 10 }), - }) - ) - ) - }) - - it("should be able to search by calculations", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - type: ViewV2Type.CALCULATION, - name: generator.guid(), - schema: { - "Quantity Sum": { - visible: true, - calculationType: CalculationType.SUM, - field: "quantity", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - expect(response.rows).toHaveLength(1) - expect(response.rows).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), - }), - ]) - ) - - // Calculation views do not return rows that can be linked back to - // the source table, and so should not have an _id field. - for (const row of response.rows) { - expect("_id" in row).toBe(false) - } - }) - - it("should be able to group by a basic field", async () => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { - visible: true, - field: "quantity", - }, - "Total Price": { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - const priceByQuantity: Record = {} - for (const row of rows) { - priceByQuantity[row.quantity] ??= 0 - priceByQuantity[row.quantity] += row.price - } - - for (const row of response.rows) { - expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) - } - }) - - it.each([ - CalculationType.COUNT, - CalculationType.SUM, - CalculationType.AVG, - CalculationType.MIN, - CalculationType.MAX, - ])("should be able to calculate $type", async type => { - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - aggregate: { - visible: true, - calculationType: type, - field: "price", - }, - }, - }) - - const response = await config.api.viewV2.search(view.id, { - query: {}, - }) - - function calculate( - type: CalculationType, - numbers: number[] - ): number { - switch (type) { - case CalculationType.COUNT: - return numbers.length - case CalculationType.SUM: - return numbers.reduce((a, b) => a + b, 0) - case CalculationType.AVG: - return numbers.reduce((a, b) => a + b, 0) / numbers.length - case CalculationType.MIN: - return Math.min(...numbers) - case CalculationType.MAX: - return Math.max(...numbers) - } - } - - const prices = rows.map(row => row.price) - const expected = calculate(type, prices) - const actual = response.rows[0].aggregate - - if (type === CalculationType.AVG) { - // The average calculation can introduce floating point rounding - // errors, so we need to compare to within a small margin of - // error. - expect(actual).toBeCloseTo(expected) - } else { - expect(actual).toEqual(expected) - } - }) - - it("should be able to do a COUNT(DISTINCT)", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - name: { - name: "name", - type: FieldType.STRING, - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "name", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "John", - }, - { - name: "John", - }, - { - name: "Sue", - }, - ], - }) - - const { rows } = await config.api.row.search(view.id) - expect(rows).toHaveLength(1) - expect(rows[0].count).toEqual(2) - }) - - it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { - await config.api.viewV2.create( - { - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - count: { - visible: true, - calculationType: CalculationType.COUNT, - distinct: true, - field: "does not exist oh no", - }, - }, - }, - { - status: 400, - body: { - message: - 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', - }, - } - ) - }) - - it("should be able to filter rows on the view itself", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - query: { - equal: { - quantity: 1, - }, - }, - schema: { - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - }) - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to filter on group by fields", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: { - equal: { - quantity: 1, - }, - }, - }) - - expect(rows).toHaveLength(1) - expect(rows[0].sum).toEqual(3) - }) - - it("should be able to sort by group by field", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "quantity", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - - it("should be able to sort by a calculation", async () => { - const table = await config.api.table.save( - saveTableRequest({ - schema: { - quantity: { - type: FieldType.NUMBER, - name: "quantity", - }, - price: { - type: FieldType.NUMBER, - name: "price", - }, - }, - }) - ) - - await config.api.row.bulkImport(table._id!, { - rows: [ - { - quantity: 1, - price: 1, - }, - { - quantity: 1, - price: 2, - }, - { - quantity: 2, - price: 10, - }, - ], - }) - - const view = await config.api.viewV2.create({ - tableId: table._id!, - name: generator.guid(), - type: ViewV2Type.CALCULATION, - schema: { - quantity: { visible: true }, - sum: { - visible: true, - calculationType: CalculationType.SUM, - field: "price", - }, - }, - }) - - const { rows } = await config.api.viewV2.search(view.id, { - query: {}, - sort: "sum", - sortOrder: SortOrder.DESCENDING, - }) - - expect(rows).toEqual([ - expect.objectContaining({ quantity: 2, sum: 10 }), - expect.objectContaining({ quantity: 1, sum: 3 }), - ]) - }) - }) - - !isLucene && - it("should not need required fields to be present", async () => { - const table = await config.api.table.save( + beforeAll(async () => { + table = await config.api.table.save( saveTableRequest({ schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", + quantity: { type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", }, }, }) ) - await Promise.all([ - config.api.row.save(table._id!, { name: "Steve", age: 30 }), - config.api.row.save(table._id!, { name: "Jane", age: 31 }), - ]) + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + it("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, - name: generator.guid(), type: ViewV2Type.CALCULATION, + name: generator.guid(), schema: { - sum: { + "Quantity Sum": { visible: true, calculationType: CalculationType.SUM, - field: "age", + field: "quantity", }, }, }) @@ -3544,9 +3023,458 @@ describe.each([ }) expect(response.rows).toHaveLength(1) - expect(response.rows[0].sum).toEqual(61) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) + ) + + // Calculation views do not return rows that can be linked back to + // the source table, and so should not have an _id field. + for (const row of response.rows) { + expect("_id" in row).toBe(false) + } }) + it("should be able to group by a basic field", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { + visible: true, + field: "quantity", + }, + "Total Price": { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + const priceByQuantity: Record = {} + for (const row of rows) { + priceByQuantity[row.quantity] ??= 0 + priceByQuantity[row.quantity] += row.price + } + + for (const row of response.rows) { + expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity]) + } + }) + + it.each([ + CalculationType.COUNT, + CalculationType.SUM, + CalculationType.AVG, + CalculationType.MIN, + CalculationType.MAX, + ])("should be able to calculate $type", async type => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + aggregate: { + visible: true, + calculationType: type, + field: "price", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + function calculate(type: CalculationType, numbers: number[]): number { + switch (type) { + case CalculationType.COUNT: + return numbers.length + case CalculationType.SUM: + return numbers.reduce((a, b) => a + b, 0) + case CalculationType.AVG: + return numbers.reduce((a, b) => a + b, 0) / numbers.length + case CalculationType.MIN: + return Math.min(...numbers) + case CalculationType.MAX: + return Math.max(...numbers) + } + } + + const prices = rows.map(row => row.price) + const expected = calculate(type, prices) + const actual = response.rows[0].aggregate + + if (type === CalculationType.AVG) { + // The average calculation can introduce floating point rounding + // errors, so we need to compare to within a small margin of + // error. + expect(actual).toBeCloseTo(expected) + } else { + expect(actual).toEqual(expected) + } + }) + + it("should be able to do a COUNT(DISTINCT)", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "name", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "John", + }, + { + name: "John", + }, + { + name: "Sue", + }, + ], + }) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(1) + expect(rows[0].count).toEqual(2) + }) + + it("should not be able to COUNT(DISTINCT ...) against a non-existent field", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "does not exist oh no", + }, + }, + }, + { + status: 400, + body: { + message: + 'Calculation field "count" references field "does not exist oh no" which does not exist in the table schema', + }, + } + ) + }) + + it("should be able to filter rows on the view itself", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + query: { + equal: { + quantity: 1, + }, + }, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + }) + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(3) + }) + + it("should be able to filter on group by fields", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: { + equal: { + quantity: 1, + }, + }, + }) + + expect(rows).toHaveLength(1) + expect(rows[0].sum).toEqual(3) + }) + + it("should be able to sort by group by field", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "quantity", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + + it("should be able to sort by a calculation", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + quantity: 1, + price: 1, + }, + { + quantity: 1, + price: 2, + }, + { + quantity: 2, + price: 10, + }, + ], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + quantity: { visible: true }, + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "price", + }, + }, + }) + + const { rows } = await config.api.viewV2.search(view.id, { + query: {}, + sort: "sum", + sortOrder: SortOrder.DESCENDING, + }) + + expect(rows).toEqual([ + expect.objectContaining({ quantity: 2, sum: 10 }), + expect.objectContaining({ quantity: 1, sum: 3 }), + ]) + }) + }) + + it("should not need required fields to be present", async () => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + }, + }) + ) + + await Promise.all([ + config.api.row.save(table._id!, { name: "Steve", age: 30 }), + config.api.row.save(table._id!, { name: "Jane", age: 31 }), + ]) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "age", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].sum).toEqual(61) + }) + it("should be able to filter on a single user field in both the view query and search query", async () => { const table = await config.api.table.save( saveTableRequest({ From b15b0fb2ae332d35039a415573654197b181e867 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:23:32 +0200 Subject: [PATCH 009/193] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 14f9c8a925..ac8eaeef29 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 14f9c8a92517bdd08ff29ad39e92cb90d4b2c02f +Subproject commit ac8eaeef29d08dfde64db6d6d30995e0179721b2 From dfbebe1c7972d7521a713a8bfc97136fd3b9a74a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:28:14 +0200 Subject: [PATCH 010/193] Cleanup sqs flags from tests --- .../src/api/routes/tests/templates.spec.ts | 65 ++++++++----------- 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/packages/server/src/api/routes/tests/templates.spec.ts b/packages/server/src/api/routes/tests/templates.spec.ts index d5483c54b4..725938cb04 100644 --- a/packages/server/src/api/routes/tests/templates.spec.ts +++ b/packages/server/src/api/routes/tests/templates.spec.ts @@ -2,7 +2,6 @@ import * as setup from "./utilities" import path from "path" import nock from "nock" import { generator } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" interface App { background: string @@ -82,48 +81,36 @@ describe("/templates", () => { }) describe("create app from template", () => { - it.each(["sqs", "lucene"])( - `should be able to create an app from a template (%s)`, - async source => { - await features.testutils.withFeatureFlags( - "*", - { SQS: source === "sqs" }, - async () => { - const name = generator.guid().replaceAll("-", "") - const url = `/${name}` + it("should be able to create an app from a template", async () => { + const name = generator.guid().replaceAll("-", "") + const url = `/${name}` - const app = await config.api.application.create({ - name, - url, - useTemplate: "true", - templateName: "Agency Client Portal", - templateKey: "app/agency-client-portal", - }) - expect(app.name).toBe(name) - expect(app.url).toBe(url) + const app = await config.api.application.create({ + name, + url, + useTemplate: "true", + templateName: "Agency Client Portal", + templateKey: "app/agency-client-portal", + }) + expect(app.name).toBe(name) + expect(app.url).toBe(url) - await config.withApp(app, async () => { - const tables = await config.api.table.fetch() - expect(tables).toHaveLength(2) + await config.withApp(app, async () => { + const tables = await config.api.table.fetch() + expect(tables).toHaveLength(2) - tables.sort((a, b) => a.name.localeCompare(b.name)) - const [agencyProjects, users] = tables - expect(agencyProjects.name).toBe("Agency Projects") - expect(users.name).toBe("Users") + tables.sort((a, b) => a.name.localeCompare(b.name)) + const [agencyProjects, users] = tables + expect(agencyProjects.name).toBe("Agency Projects") + expect(users.name).toBe("Users") - const { rows } = await config.api.row.search( - agencyProjects._id!, - { - tableId: agencyProjects._id!, - query: {}, - } - ) + const { rows } = await config.api.row.search(agencyProjects._id!, { + tableId: agencyProjects._id!, + query: {}, + }) - expect(rows).toHaveLength(3) - }) - } - ) - } - ) + expect(rows).toHaveLength(3) + }) + }) }) }) From 1a572036f7cfa61305194471e3782a4d97925bee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:28:19 +0200 Subject: [PATCH 011/193] Cleanup sqs flags from tests --- .../src/api/routes/tests/application.spec.ts | 17 +---------------- .../server/src/api/routes/tests/row.spec.ts | 4 +--- .../server/src/api/routes/tests/search.spec.ts | 5 +---- 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts index fe8250bde5..3f088d2705 100644 --- a/packages/server/src/api/routes/tests/application.spec.ts +++ b/packages/server/src/api/routes/tests/application.spec.ts @@ -14,7 +14,7 @@ jest.mock("../../../utilities/redis", () => ({ import { checkBuilderEndpoint } from "./utilities/TestFunctions" import * as setup from "./utilities" import { AppStatus } from "../../../db/utils" -import { events, utils, context, features } from "@budibase/backend-core" +import { events, utils, context } from "@budibase/backend-core" import env from "../../../environment" import { type App } from "@budibase/types" import tk from "timekeeper" @@ -346,21 +346,6 @@ describe("/applications", () => { expect(events.app.deleted).toHaveBeenCalledTimes(1) expect(events.app.unpublished).toHaveBeenCalledTimes(1) }) - - it("should be able to delete an app after SQS has been set but app hasn't been migrated", async () => { - const prodAppId = app.appId.replace("_dev", "") - nock("http://localhost:10000") - .delete(`/api/global/roles/${prodAppId}`) - .reply(200, {}) - - await features.testutils.withFeatureFlags( - "*", - { SQS: true }, - async () => { - await config.api.application.delete(app.appId) - } - ) - }) }) describe("POST /api/applications/:appId/duplicate", () => { diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index bd8fbb8c79..5150ed42d9 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -90,9 +90,7 @@ describe.each([ let client: Knex | undefined beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: true }, () => - config.init() - ) + await config.init() if (dsProvider) { const rawDatasource = await dsProvider diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 11594f5d82..48d9eaeaf1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -7,7 +7,6 @@ import { import { context, db as dbCore, - features, MAX_VALID_DATE, MIN_VALID_DATE, SQLITE_DESIGN_DOC_ID, @@ -91,9 +90,7 @@ describe.each([ } beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: true }, () => - config.init() - ) + await config.init() if (config.app?.appId) { config.app = await config.api.application.update(config.app?.appId, { From e7d4f90f1beb9280f87d49c58546fa2d0ae51e79 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:31:33 +0200 Subject: [PATCH 012/193] Cleanup sqs flags from tests --- .../sdk/app/rows/search/tests/search.spec.ts | 78 ++++++++----------- .../api/routes/global/tests/auditLogs.spec.ts | 9 +-- 2 files changed, 34 insertions(+), 53 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 4d8a6b6d69..cf91033c40 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -10,7 +10,7 @@ import { import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" import { generator } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" + import { DatabaseName, getDatasource, @@ -21,30 +21,20 @@ import { tableForDatasource } from "../../../../../tests/utilities/structures" // (e.g. limiting searches to returning specific fields). If it's possible to // test through the API, it should be done there instead. describe.each([ - ["lucene", undefined], - ["sqs", undefined], + ["internal", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("search sdk (%s)", (name, dsProvider) => { - const isSqs = name === "sqs" - const isLucene = name === "lucene" - const isInternal = isLucene || isSqs + const isInternal = name === "internal" const config = new TestConfiguration() - let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let table: Table beforeAll(async () => { - await features.testutils.withFeatureFlags("*", { SQS: isSqs }, () => - config.init() - ) - - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: isSqs, - }) + await config.init() if (dsProvider) { datasource = await config.createDatasource({ @@ -105,9 +95,6 @@ describe.each([ afterAll(async () => { config.end() - if (envCleanup) { - envCleanup() - } }) it("querying by fields will always return data attribute columns", async () => { @@ -211,36 +198,35 @@ describe.each([ }) }) - !isLucene && - it.each([ - [["id", "name", "age"], 3], - [["name", "age"], 10], - ])( - "cannot query by non search fields (fields: %s)", - async (queryFields, expectedRows) => { - await config.doInContext(config.appId, async () => { - const { rows } = await search({ - tableId: table._id!, - query: { - $or: { - conditions: [ - { - $and: { - conditions: [ - { range: { id: { low: 2, high: 4 } } }, - { range: { id: { low: 3, high: 5 } } }, - ], - }, + it.each([ + [["id", "name", "age"], 3], + [["name", "age"], 10], + ])( + "cannot query by non search fields (fields: %s)", + async (queryFields, expectedRows) => { + await config.doInContext(config.appId, async () => { + const { rows } = await search({ + tableId: table._id!, + query: { + $or: { + conditions: [ + { + $and: { + conditions: [ + { range: { id: { low: 2, high: 4 } } }, + { range: { id: { low: 3, high: 5 } } }, + ], }, - { equal: { id: 7 } }, - ], - }, + }, + { equal: { id: 7 } }, + ], }, - fields: queryFields, - }) - - expect(rows).toHaveLength(expectedRows) + }, + fields: queryFields, }) - } - ) + + expect(rows).toHaveLength(expectedRows) + }) + } + ) }) diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index b540836583..f901925016 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -1,5 +1,5 @@ import { mocks, structures } from "@budibase/backend-core/tests" -import { context, events, features } from "@budibase/backend-core" +import { context, events } from "@budibase/backend-core" import { Event, IdentityType } from "@budibase/types" import { TestConfiguration } from "../../../../tests" @@ -12,19 +12,14 @@ const BASE_IDENTITY = { const USER_AUDIT_LOG_COUNT = 3 const APP_ID = "app_1" -describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { +describe("/api/global/auditlogs (%s)", () => { const config = new TestConfiguration() - let envCleanup: (() => void) | undefined beforeAll(async () => { - envCleanup = features.testutils.setFeatureFlags("*", { - SQS: method === "sql", - }) await config.beforeAll() }) afterAll(async () => { - envCleanup?.() await config.afterAll() }) From aeeb4b6b0f2b9802a7d9702e11bcddc46789febe Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:37:42 +0200 Subject: [PATCH 013/193] Cleanup sqs flags from tests --- .../tests/20240604153647_initial_sqs.spec.ts | 89 +++++++------------ .../tests/outputProcessing.spec.ts | 8 +- 2 files changed, 33 insertions(+), 64 deletions(-) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index fe44b7b901..1ce519b0b0 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -1,10 +1,6 @@ import * as setup from "../../../api/routes/tests/utilities" import { basicTable } from "../../../tests/utilities/structures" -import { - db as dbCore, - features, - SQLITE_DESIGN_DOC_ID, -} from "@budibase/backend-core" +import { db as dbCore, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core" import { LinkDocument, DocumentType, @@ -70,24 +66,14 @@ function oldLinkDocument(): Omit { } } -async function sqsDisabled(cb: () => Promise) { - await features.testutils.withFeatureFlags("*", { SQS: false }, cb) -} - -async function sqsEnabled(cb: () => Promise) { - await features.testutils.withFeatureFlags("*", { SQS: true }, cb) -} - describe("SQS migration", () => { beforeAll(async () => { - await sqsDisabled(async () => { - await config.init() - const table = await config.api.table.save(basicTable()) - tableId = table._id! - const db = dbCore.getDB(config.appId!) - // old link document - await db.put(oldLinkDocument()) - }) + await config.init() + const table = await config.api.table.save(basicTable()) + tableId = table._id! + const db = dbCore.getDB(config.appId!) + // old link document + await db.put(oldLinkDocument()) }) beforeEach(async () => { @@ -101,43 +87,32 @@ describe("SQS migration", () => { it("test migration runs as expected against an older DB", async () => { const db = dbCore.getDB(config.appId!) - // confirm nothing exists initially - await sqsDisabled(async () => { - let error: any | undefined - try { - await db.get(SQLITE_DESIGN_DOC_ID) - } catch (err: any) { - error = err - } - expect(error).toBeDefined() - expect(error.status).toBe(404) + + // remove sqlite design doc to simulate it comes from an older installation + const doc = await db.get(SQLITE_DESIGN_DOC_ID) + await db.remove({ _id: doc._id, _rev: doc._rev }) + + await processMigrations(config.appId!, MIGRATIONS) + const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) + expect(designDoc.sql.tables).toBeDefined() + const mainTableDef = designDoc.sql.tables[tableId] + expect(mainTableDef).toBeDefined() + expect(mainTableDef.fields[prefix("name")]).toEqual({ + field: "name", + type: SQLiteType.TEXT, + }) + expect(mainTableDef.fields[prefix("description")]).toEqual({ + field: "description", + type: SQLiteType.TEXT, }) - await sqsEnabled(async () => { - await processMigrations(config.appId!, MIGRATIONS) - const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) - expect(designDoc.sql.tables).toBeDefined() - const mainTableDef = designDoc.sql.tables[tableId] - expect(mainTableDef).toBeDefined() - expect(mainTableDef.fields[prefix("name")]).toEqual({ - field: "name", - type: SQLiteType.TEXT, - }) - expect(mainTableDef.fields[prefix("description")]).toEqual({ - field: "description", - type: SQLiteType.TEXT, - }) - - const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() - const linkDoc = await db.get(oldLinkDocID()) - expect(linkDoc.tableId).toEqual( - generateJunctionTableID(tableId1, tableId2) - ) - // should have swapped the documents - expect(linkDoc.doc1.tableId).toEqual(tableId2) - expect(linkDoc.doc1.rowId).toEqual(rowId2) - expect(linkDoc.doc2.tableId).toEqual(tableId1) - expect(linkDoc.doc2.rowId).toEqual(rowId1) - }) + const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() + const linkDoc = await db.get(oldLinkDocID()) + expect(linkDoc.tableId).toEqual(generateJunctionTableID(tableId1, tableId2)) + // should have swapped the documents + expect(linkDoc.doc1.tableId).toEqual(tableId2) + expect(linkDoc.doc1.rowId).toEqual(rowId2) + expect(linkDoc.doc2.tableId).toEqual(tableId1) + expect(linkDoc.doc2.rowId).toEqual(rowId1) }) }) diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts index 8cbe585d90..cd375ecb23 100644 --- a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -8,7 +8,7 @@ import { } from "@budibase/types" import { outputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" -import { features } from "@budibase/backend-core" + import * as bbReferenceProcessor from "../bbReferenceProcessor" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -21,7 +21,6 @@ jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ describe("rowProcessor - outputProcessing", () => { const config = new TestConfiguration() - let cleanupFlags: () => void = () => {} beforeAll(async () => { await config.init() @@ -33,11 +32,6 @@ describe("rowProcessor - outputProcessing", () => { beforeEach(() => { jest.resetAllMocks() - cleanupFlags = features.testutils.setFeatureFlags("*", { SQS: true }) - }) - - afterEach(() => { - cleanupFlags() }) const processOutputBBReferenceMock = From 630802799e731bc618d6d9f36db41aab17c6b10c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 15:40:10 +0200 Subject: [PATCH 014/193] Remove SQS flag --- packages/backend-core/src/features/features.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index e2f8d9b6a1..aa64480d5d 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,6 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ [FeatureFlag.DEFAULT_VALUES]: Flag.boolean(env.isDev()), [FeatureFlag.AUTOMATION_BRANCHING]: Flag.boolean(env.isDev()), - [FeatureFlag.SQS]: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), From ff14191af77ceb19729063511051d0559562f654 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 21 Oct 2024 16:01:12 +0200 Subject: [PATCH 015/193] Fix test --- packages/server/src/api/routes/tests/viewV2.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index fe44b495e3..8dfd4df33e 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -93,6 +93,7 @@ describe.each([ } beforeAll(async () => { + await config.init() if (dsProvider) { datasource = await config.createDatasource({ datasource: await dsProvider, From f8f9c4da8efc6ec94e1dd52fe525fd9cbfdeb7af Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 25 Oct 2024 17:32:42 +0100 Subject: [PATCH 016/193] Work in progress - this is the scaffolding for views to be added to the public API. --- packages/server/specs/resources/view.ts | 193 ++++++++++++++++++ .../src/api/controllers/public/views.ts | 56 +++++ .../server/src/api/routes/public/views.ts | 165 +++++++++++++++ .../server/src/api/routes/utils/validators.ts | 4 + 4 files changed, 418 insertions(+) create mode 100644 packages/server/specs/resources/view.ts create mode 100644 packages/server/src/api/controllers/public/views.ts create mode 100644 packages/server/src/api/routes/public/views.ts diff --git a/packages/server/specs/resources/view.ts b/packages/server/specs/resources/view.ts new file mode 100644 index 0000000000..6458af15fc --- /dev/null +++ b/packages/server/specs/resources/view.ts @@ -0,0 +1,193 @@ +import { FieldType, FormulaType, RelationshipType } from "@budibase/types" +import { object } from "./utils" +import Resource from "./utils/Resource" + +const view = { + _id: "ta_5b1649e42a5b41dea4ef7742a36a7a70", + name: "People", + schema: { + name: { + type: "string", + name: "name", + }, + age: { + type: "number", + name: "age", + }, + relationship: { + type: "link", + name: "relationship", + tableId: "ta_...", + fieldName: "relatedColumn", + relationshipType: "many-to-many", + }, + }, +} + +const baseColumnDef = { + type: { + type: "string", + enum: Object.values(FieldType), + description: + "Defines the type of the column, most explain themselves, a link column is a relationship.", + }, + constraints: { + type: "object", + description: + "A constraint can be applied to the column which will be validated against when a row is saved.", + properties: { + type: { + type: "string", + enum: ["string", "number", "object", "boolean"], + }, + presence: { + type: "boolean", + description: "Defines whether the column is required or not.", + }, + }, + }, + name: { + type: "string", + description: "The name of the column.", + }, + autocolumn: { + type: "boolean", + description: "Defines whether the column is automatically generated.", + }, +} + +const viewSchema = { + description: "The table to be created/updated.", + type: "object", + required: ["name", "schema"], + properties: { + name: { + description: "The name of the table.", + type: "string", + }, + primaryDisplay: { + type: "string", + description: + "The name of the column which should be used in relationship tags when relating to this table.", + }, + schema: { + type: "object", + additionalProperties: { + oneOf: [ + // relationship + { + type: "object", + properties: { + ...baseColumnDef, + type: { + type: "string", + enum: [FieldType.LINK], + description: "A relationship column.", + }, + fieldName: { + type: "string", + description: + "The name of the column which a relationship column is related to in another table.", + }, + tableId: { + type: "string", + description: + "The ID of the table which a relationship column is related to.", + }, + relationshipType: { + type: "string", + enum: Object.values(RelationshipType), + description: + "Defines the type of relationship that this column will be used for.", + }, + through: { + type: "string", + description: + "When using a SQL table that contains many to many relationships this defines the table the relationships are linked through.", + }, + foreignKey: { + type: "string", + description: + "When using a SQL table that contains a one to many relationship this defines the foreign key.", + }, + throughFrom: { + type: "string", + description: + "When using a SQL table that utilises a through table, this defines the primary key in the through table for this table.", + }, + throughTo: { + type: "string", + description: + "When using a SQL table that utilises a through table, this defines the primary key in the through table for the related table.", + }, + }, + }, + { + type: "object", + properties: { + ...baseColumnDef, + type: { + type: "string", + enum: [FieldType.FORMULA], + description: "A formula column.", + }, + formula: { + type: "string", + description: + "Defines a Handlebars or JavaScript formula to use, note that Javascript formulas are expected to be provided in the base64 format.", + }, + formulaType: { + type: "string", + enum: Object.values(FormulaType), + description: + "Defines whether this is a static or dynamic formula.", + }, + }, + }, + { + type: "object", + properties: baseColumnDef, + }, + ], + }, + }, + }, +} + +const viewOutputSchema = { + ...viewSchema, + properties: { + ...viewSchema.properties, + _id: { + description: "The ID of the view.", + type: "string", + }, + }, + required: [...viewSchema.required, "_id"], +} + +export default new Resource() + .setExamples({ + view: { + value: { + data: view, + }, + }, + views: { + value: { + data: [view], + }, + }, + }) + .setSchemas({ + view: viewSchema, + viewOutput: object({ + data: viewOutputSchema, + }), + viewSearch: object({ + data: { + type: "array", + items: viewOutputSchema, + }, + }), + }) diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts new file mode 100644 index 0000000000..11fb4cc344 --- /dev/null +++ b/packages/server/src/api/controllers/public/views.ts @@ -0,0 +1,56 @@ +import { search as stringSearch } from "./utils" +import * as controller from "../view" +import { ViewV2, UserCtx } from "@budibase/types" +import { Next } from "koa" + +function fixView(view: ViewV2, params: any) { + if (!params || !view) { + return view + } + if (params.viewId) { + view.id = params.viewId + } + return view +} + +export async function search(ctx: UserCtx, next: Next) { + const { name } = ctx.request.body + // TODO: need a view search endpoint + // await controller.v2.fetch(ctx) + ctx.body = stringSearch(ctx.body, name) + await next() +} + +export async function create(ctx: UserCtx, next: Next) { + await controller.v2.create(ctx) + await next() +} + +export async function read(ctx: UserCtx, next: Next) { + await controller.v2.get(ctx) + await next() +} + +export async function update(ctx: UserCtx, next: Next) { + // TODO: this is more complex - no rev on views + // ctx.request.body = await addRev( + // fixView(ctx.request.body, ctx.params), + // ctx.params.tableId + // ) + await controller.v2.update(ctx) + await next() +} + +export async function destroy(ctx: UserCtx, next: Next) { + await controller.v2.remove(ctx) + ctx.body = ctx.table + await next() +} + +export default { + create, + read, + update, + destroy, + search, +} diff --git a/packages/server/src/api/routes/public/views.ts b/packages/server/src/api/routes/public/views.ts new file mode 100644 index 0000000000..7c182d105f --- /dev/null +++ b/packages/server/src/api/routes/public/views.ts @@ -0,0 +1,165 @@ +import controller from "../../controllers/public/views" +import Endpoint from "./utils/Endpoint" +import { viewValidator, nameValidator } from "../utils/validators" + +const read = [], + write = [] + +/** + * @openapi + * /views: + * post: + * operationId: viewCreate + * summary: Create a view + * description: Create a view, this can be against an internal or external table. + * tags: + * - views + * parameters: + * - $ref: '#/components/parameters/appId' + * requestBody: + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/view' + * examples: + * view: + * $ref: '#/components/examples/view' + * responses: + * 200: + * description: Returns the created view, including the ID which has been generated for it. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewOutput' + * examples: + * view: + * $ref: '#/components/examples/view' + */ +write.push( + new Endpoint("post", "/views", controller.create).addMiddleware( + viewValidator() + ) +) + +/** + * @openapi + * /views/{viewId}: + * put: + * operationId: viewUpdate + * summary: Update a view + * description: Update a view, this can be against an internal or external table. + * tags: + * - views + * parameters: + * - $ref: '#/components/parameters/viewId' + * - $ref: '#/components/parameters/appId' + * requestBody: + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/view' + * examples: + * view: + * $ref: '#/components/examples/view' + * responses: + * 200: + * description: Returns the updated view. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewOutput' + * examples: + * view: + * $ref: '#/components/examples/view' + */ +write.push( + new Endpoint("put", "/views/:viewId", controller.update).addMiddleware( + viewValidator() + ) +) + +/** + * @openapi + * /views/{viewId}: + * delete: + * operationId: viewDestroy + * summary: Delete a view + * description: Delete a view, this can be against an internal or external table. + * tags: + * - views + * parameters: + * - $ref: '#/components/parameters/viewId' + * - $ref: '#/components/parameters/appId' + * responses: + * 200: + * description: Returns the deleted view. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewOutput' + * examples: + * view: + * $ref: '#/components/examples/view' + */ +write.push(new Endpoint("delete", "/views/:viewId", controller.destroy)) + +/** + * @openapi + * /views/{viewId}: + * get: + * operationId: viewGetById + * summary: Retrieve a view + * description: Lookup a view, this could be internal or external. + * tags: + * - views + * parameters: + * - $ref: '#/components/parameters/viewId' + * - $ref: '#/components/parameters/appId' + * responses: + * 200: + * description: Returns the retrieved view. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewOutput' + * examples: + * view: + * $ref: '#/components/examples/view' + */ +read.push(new Endpoint("get", "/views/:viewId", controller.read)) + +/** + * @openapi + * /views/search: + * post: + * operationId: viewSearch + * summary: Search for views + * description: Based on view properties (currently only name) search for views. + * tags: + * - views + * parameters: + * - $ref: '#/components/parameters/appId' + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewSearch' + * responses: + * 200: + * description: Returns the found views, based on the search parameters. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/viewSearch' + * examples: + * views: + * $ref: '#/components/examples/views' + */ +read.push( + new Endpoint("post", "/views/search", controller.search).addMiddleware( + nameValidator() + ) +) + +export default { read, write } diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 862d8c30c5..68ebd72c5e 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -66,6 +66,10 @@ export function tableValidator() { ) } +export function viewValidator() { + return auth.joiValidator.body(Joi.object()) +} + export function nameValidator() { return auth.joiValidator.body( Joi.object({ From eeb78b2c4578e1a6a1b181e7e1377af8b38e47ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 25 Oct 2024 17:51:52 +0100 Subject: [PATCH 017/193] Getting schemas correct for views. --- packages/server/specs/resources/view.ts | 160 +++++------------- .../src/api/controllers/public/views.ts | 8 +- 2 files changed, 45 insertions(+), 123 deletions(-) diff --git a/packages/server/specs/resources/view.ts b/packages/server/specs/resources/view.ts index 6458af15fc..5afdf78a0e 100644 --- a/packages/server/specs/resources/view.ts +++ b/packages/server/specs/resources/view.ts @@ -1,154 +1,72 @@ -import { FieldType, FormulaType, RelationshipType } from "@budibase/types" import { object } from "./utils" import Resource from "./utils/Resource" const view = { - _id: "ta_5b1649e42a5b41dea4ef7742a36a7a70", - name: "People", + name: "peopleView", + tableId: "ta_896a325f7e8147d2a2cda93c5d236511", schema: { name: { - type: "string", - name: "name", + visible: true, + readonly: false, + order: 1, + width: 300, }, age: { - type: "number", - name: "age", + visible: true, + readonly: true, + order: 2, + width: 200, }, - relationship: { - type: "link", - name: "relationship", - tableId: "ta_...", - fieldName: "relatedColumn", - relationshipType: "many-to-many", + salary: { + visible: false, + readonly: false, }, }, + primaryDisplay: "name", } const baseColumnDef = { - type: { - type: "string", - enum: Object.values(FieldType), - description: - "Defines the type of the column, most explain themselves, a link column is a relationship.", - }, - constraints: { - type: "object", - description: - "A constraint can be applied to the column which will be validated against when a row is saved.", - properties: { - type: { - type: "string", - enum: ["string", "number", "object", "boolean"], - }, - presence: { - type: "boolean", - description: "Defines whether the column is required or not.", - }, - }, - }, - name: { - type: "string", - description: "The name of the column.", - }, - autocolumn: { + visible: { type: "boolean", - description: "Defines whether the column is automatically generated.", + description: + "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it.", + }, + readonly: { + type: "boolean", + description: + "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated.", + }, + order: { + type: "integer", + description: + "A number defining where the column shows up in tables, lowest being first.", + }, + width: { + type: "integer", + description: + "A width for the column, defined in pixels - this affects rendering in tables.", }, } const viewSchema = { - description: "The table to be created/updated.", + description: "The view to be created/updated.", type: "object", required: ["name", "schema"], properties: { name: { - description: "The name of the table.", + description: "The name of the view.", type: "string", }, primaryDisplay: { type: "string", description: - "The name of the column which should be used in relationship tags when relating to this table.", + "A column used to display rows from this view - usually used when rendered in tables.", }, schema: { type: "object", additionalProperties: { - oneOf: [ - // relationship - { - type: "object", - properties: { - ...baseColumnDef, - type: { - type: "string", - enum: [FieldType.LINK], - description: "A relationship column.", - }, - fieldName: { - type: "string", - description: - "The name of the column which a relationship column is related to in another table.", - }, - tableId: { - type: "string", - description: - "The ID of the table which a relationship column is related to.", - }, - relationshipType: { - type: "string", - enum: Object.values(RelationshipType), - description: - "Defines the type of relationship that this column will be used for.", - }, - through: { - type: "string", - description: - "When using a SQL table that contains many to many relationships this defines the table the relationships are linked through.", - }, - foreignKey: { - type: "string", - description: - "When using a SQL table that contains a one to many relationship this defines the foreign key.", - }, - throughFrom: { - type: "string", - description: - "When using a SQL table that utilises a through table, this defines the primary key in the through table for this table.", - }, - throughTo: { - type: "string", - description: - "When using a SQL table that utilises a through table, this defines the primary key in the through table for the related table.", - }, - }, - }, - { - type: "object", - properties: { - ...baseColumnDef, - type: { - type: "string", - enum: [FieldType.FORMULA], - description: "A formula column.", - }, - formula: { - type: "string", - description: - "Defines a Handlebars or JavaScript formula to use, note that Javascript formulas are expected to be provided in the base64 format.", - }, - formulaType: { - type: "string", - enum: Object.values(FormulaType), - description: - "Defines whether this is a static or dynamic formula.", - }, - }, - }, - { - type: "object", - properties: baseColumnDef, - }, - ], + type: "object", + properties: baseColumnDef, }, }, }, @@ -158,12 +76,12 @@ const viewOutputSchema = { ...viewSchema, properties: { ...viewSchema.properties, - _id: { + id: { description: "The ID of the view.", type: "string", }, }, - required: [...viewSchema.required, "_id"], + required: [...viewSchema.required, "id"], } export default new Resource() diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts index 11fb4cc344..85979126ff 100644 --- a/packages/server/src/api/controllers/public/views.ts +++ b/packages/server/src/api/controllers/public/views.ts @@ -3,13 +3,16 @@ import * as controller from "../view" import { ViewV2, UserCtx } from "@budibase/types" import { Next } from "koa" -function fixView(view: ViewV2, params: any) { +function fixView(view: ViewV2, params?: { viewId: string }) { if (!params || !view) { return view } - if (params.viewId) { + if (params?.viewId) { view.id = params.viewId } + if (!view.version) { + view.version = 2 + } return view } @@ -22,6 +25,7 @@ export async function search(ctx: UserCtx, next: Next) { } export async function create(ctx: UserCtx, next: Next) { + ctx.body = fixView(ctx.body) await controller.v2.create(ctx) await next() } From c055e2ae729943f10cb697fe51c41e32bf89e457 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 28 Oct 2024 18:09:04 +0000 Subject: [PATCH 018/193] Adding fetch endpoint for views (will be used by search for views in Public API). --- .../src/api/controllers/view/viewsV2.ts | 7 +++++ packages/server/src/api/routes/view.ts | 5 ++++ packages/server/src/sdk/app/views/index.ts | 29 +++++++++++++++++-- packages/types/src/api/web/app/view.ts | 4 +++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index f864df9e9e..6f92e7399c 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -12,6 +12,7 @@ import { RelationSchemaField, ViewFieldMetadata, CalculationType, + ViewFetchResponseEnriched, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" import { helpers } from "@budibase/shared-core" @@ -119,6 +120,12 @@ export async function get(ctx: Ctx) { } } +export async function fetch(ctx: Ctx) { + ctx.body = { + data: await sdk.views.getAllEnriched(), + } +} + export async function create(ctx: Ctx) { const view = ctx.request.body const { tableId } = view diff --git a/packages/server/src/api/routes/view.ts b/packages/server/src/api/routes/view.ts index 807d8e2f28..92e5f02a18 100644 --- a/packages/server/src/api/routes/view.ts +++ b/packages/server/src/api/routes/view.ts @@ -8,6 +8,11 @@ import { permissions } from "@budibase/backend-core" const router: Router = new Router() router + .get( + "/api/v2/views", + authorized(permissions.BUILDER), + viewController.v2.fetch + ) .get( "/api/v2/views/:viewId", authorizedResource( diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index cbf3513dd0..fc2341722f 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -27,6 +27,7 @@ import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./internal" import * as external from "./external" import sdk from "../../../sdk" +import { ensureQueryUISet } from "./utils" function pickApi(tableId: any) { if (isExternalTableID(tableId)) { @@ -45,6 +46,24 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +export async function getAllEnriched(): Promise { + const tables = await sdk.tables.getAllTables() + let views: ViewV2Enriched[] = [] + for (let table of tables) { + if (!table.views || Object.keys(table.views).length === 0) { + continue + } + const v2Views = Object.values(table.views).filter(isV2) + const enrichedViews = await Promise.all( + v2Views.map(view => + enrichSchema(ensureQueryUISet(view), table.schema, tables) + ) + ) + views = views.concat(enrichedViews) + } + return views +} + export async function getTable(view: string | ViewV2): Promise
{ const viewId = typeof view === "string" ? view : view.id const cached = context.getTableForView(viewId) @@ -303,13 +322,19 @@ export function allowedFields( export async function enrichSchema( view: ViewV2, - tableSchema: TableSchema + tableSchema: TableSchema, + tables?: Table[] ): Promise { async function populateRelTableSchema( tableId: string, viewFields: Record ) { - const relTable = await sdk.tables.getTable(tableId) + let relTable = tables + ? tables?.find(t => t._id === tableId) + : await sdk.tables.getTable(tableId) + if (!relTable) { + throw new Error("Cannot enrich relationship, table not found") + } const result: Record = {} for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..2560f7507f 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,10 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } +export interface ViewFetchResponseEnriched { + data: ViewV2Enriched[] +} + export interface CreateViewRequest extends Omit {} export interface UpdateViewRequest extends ViewV2 {} From 06984548556d00993aba791271c9f4a7e6dce5b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 28 Oct 2024 18:09:22 +0000 Subject: [PATCH 019/193] Fixing an issue with getAllExternalTables getter also returning internal tables. --- packages/server/src/sdk/app/tables/getters.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a8ad606647..ebf7b3547b 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -82,8 +82,11 @@ export async function getAllInternalTables(db?: Database): Promise { } async function getAllExternalTables(): Promise { + // this is all datasources, we'll need to filter out internal const datasources = await sdk.datasources.fetch({ enriched: true }) - const allEntities = datasources.map(datasource => datasource.entities) + const allEntities = datasources + .filter(datasource => datasource._id !== INTERNAL_TABLE_SOURCE_ID) + .map(datasource => datasource.entities) let final: Table[] = [] for (let entities of allEntities) { if (entities) { From 68a7f88db667ad30dfa0998dae4c0b62a049589e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 28 Oct 2024 18:09:38 +0000 Subject: [PATCH 020/193] Adding test case for fetch. --- .../src/api/routes/tests/viewV2.spec.ts | 32 +++++++++++++++++++ .../server/src/tests/utilities/api/viewV2.ts | 7 ++++ 2 files changed, 39 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 2af11b513b..b344e50bb4 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2592,6 +2592,38 @@ describe.each([ }) }) + describe("fetch", () => { + let view: ViewV2, view2: ViewV2 + let table: Table, table2: Table + + beforeEach(async () => { + // clean app to make sure fixed amount of views + await config.init() + table = await config.api.table.save(saveTableRequest()) + table2 = await config.api.table.save(saveTableRequest()) + + view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: {}, + }) + view2 = await config.api.viewV2.create({ + tableId: table2._id!, + name: generator.guid(), + schema: {}, + }) + }) + + it("should be able to list views", async () => { + const response = await config.api.viewV2.fetch({ + status: 200, + }) + expect(response.data.length).toEqual(2) + expect(response.data.find(v => v.id === view.id)).toBeDefined() + expect(response.data.find(v => v.id === view2.id)).toBeDefined() + }) + }) + describe("read", () => { let view: ViewV2 let table: Table diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index 9741240f27..7cc57673a0 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -5,6 +5,7 @@ import { SearchViewRowRequest, PaginatedSearchRowResponse, ViewResponseEnriched, + ViewFetchResponseEnriched, } from "@budibase/types" import { Expectations, TestAPI } from "./base" @@ -49,6 +50,12 @@ export class ViewV2API extends TestAPI { .data } + fetch = async (expectations?: Expectations) => { + return await this._get(`/api/v2/views`, { + expectations, + }) + } + search = async ( viewId: string, params?: SearchViewRowRequest, From dd0764eb600f7c47e6ace83eb94a347b58a4a58d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 30 Oct 2024 14:08:03 +0000 Subject: [PATCH 021/193] Putting view mapping in place, updating openAPI spec. --- packages/server/specs/openapi.json | 423 +++++++++++++++++- packages/server/specs/openapi.yaml | 290 +++++++++++- packages/server/specs/resources/index.ts | 4 + .../api/controllers/public/mapping/index.ts | 2 + .../api/controllers/public/mapping/types.ts | 3 + .../api/controllers/public/mapping/views.ts | 26 ++ .../src/api/controllers/public/views.ts | 3 +- .../api/routes/public/middleware/mapper.ts | 30 +- packages/server/src/definitions/openapi.ts | 180 ++++++++ 9 files changed, 943 insertions(+), 18 deletions(-) create mode 100644 packages/server/src/api/controllers/public/mapping/views.ts diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index f3091a1fc7..c80f947fad 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -423,6 +423,62 @@ }, "metrics": { "value": "# HELP budibase_os_uptime Time in seconds that the host operating system has been up.\n# TYPE budibase_os_uptime counter\nbudibase_os_uptime 54958\n# HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.\n# TYPE budibase_os_free_mem gauge\nbudibase_os_free_mem 804507648\n# HELP budibase_os_total_mem Total bytes of memory on the host operating system.\n# TYPE budibase_os_total_mem gauge\nbudibase_os_total_mem 16742404096\n# HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.\n# TYPE budibase_os_used_mem gauge\nbudibase_os_used_mem 15937896448\n# HELP budibase_os_load1 Host operating system load average.\n# TYPE budibase_os_load1 gauge\nbudibase_os_load1 1.91\n# HELP budibase_os_load5 Host operating system load average.\n# TYPE budibase_os_load5 gauge\nbudibase_os_load5 1.75\n# HELP budibase_os_load15 Host operating system load average.\n# TYPE budibase_os_load15 gauge\nbudibase_os_load15 1.56\n# HELP budibase_tenant_user_count The number of users created.\n# TYPE budibase_tenant_user_count gauge\nbudibase_tenant_user_count 1\n# HELP budibase_tenant_app_count The number of apps created by a user.\n# TYPE budibase_tenant_app_count gauge\nbudibase_tenant_app_count 2\n# HELP budibase_tenant_production_app_count The number of apps a user has published.\n# TYPE budibase_tenant_production_app_count gauge\nbudibase_tenant_production_app_count 1\n# HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.\n# TYPE budibase_tenant_dev_app_count gauge\nbudibase_tenant_dev_app_count 1\n# HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.\n# TYPE budibase_tenant_db_count gauge\nbudibase_tenant_db_count 3\n# HELP budibase_quota_usage_apps The number of apps created.\n# TYPE budibase_quota_usage_apps gauge\nbudibase_quota_usage_apps 1\n# HELP budibase_quota_limit_apps The limit on the number of apps that can be created.\n# TYPE budibase_quota_limit_apps gauge\nbudibase_quota_limit_apps 9007199254740991\n# HELP budibase_quota_usage_rows The number of database rows used from the quota.\n# TYPE budibase_quota_usage_rows gauge\nbudibase_quota_usage_rows 0\n# HELP budibase_quota_limit_rows The limit on the number of rows that can be created.\n# TYPE budibase_quota_limit_rows gauge\nbudibase_quota_limit_rows 9007199254740991\n# HELP budibase_quota_usage_plugins The number of plugins in use.\n# TYPE budibase_quota_usage_plugins gauge\nbudibase_quota_usage_plugins 0\n# HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.\n# TYPE budibase_quota_limit_plugins gauge\nbudibase_quota_limit_plugins 9007199254740991\n# HELP budibase_quota_usage_user_groups The number of user groups created.\n# TYPE budibase_quota_usage_user_groups gauge\nbudibase_quota_usage_user_groups 0\n# HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.\n# TYPE budibase_quota_limit_user_groups gauge\nbudibase_quota_limit_user_groups 9007199254740991\n# HELP budibase_quota_usage_queries The number of queries used in the current month.\n# TYPE budibase_quota_usage_queries gauge\nbudibase_quota_usage_queries 0\n# HELP budibase_quota_limit_queries The limit on the number of queries for the current month.\n# TYPE budibase_quota_limit_queries gauge\nbudibase_quota_limit_queries 9007199254740991\n# HELP budibase_quota_usage_automations The number of automations used in the current month.\n# TYPE budibase_quota_usage_automations gauge\nbudibase_quota_usage_automations 0\n# HELP budibase_quota_limit_automations The limit on the number of automations that can be created.\n# TYPE budibase_quota_limit_automations gauge\nbudibase_quota_limit_automations 9007199254740991\n" + }, + "view": { + "value": { + "data": { + "name": "peopleView", + "tableId": "ta_896a325f7e8147d2a2cda93c5d236511", + "schema": { + "name": { + "visible": true, + "readonly": false, + "order": 1, + "width": 300 + }, + "age": { + "visible": true, + "readonly": true, + "order": 2, + "width": 200 + }, + "salary": { + "visible": false, + "readonly": false + } + }, + "primaryDisplay": "name" + } + } + }, + "views": { + "value": { + "data": [ + { + "name": "peopleView", + "tableId": "ta_896a325f7e8147d2a2cda93c5d236511", + "schema": { + "name": { + "visible": true, + "readonly": false, + "order": 1, + "width": 300 + }, + "age": { + "visible": true, + "readonly": true, + "order": 2, + "width": 200 + }, + "salary": { + "visible": false, + "readonly": false + } + }, + "primaryDisplay": "name" + } + ] + } } }, "securitySchemes": { @@ -831,8 +887,7 @@ "type": "string", "enum": [ "static", - "dynamic", - "ai" + "dynamic" ], "description": "Defines whether this is a static or dynamic formula." } @@ -1042,8 +1097,7 @@ "type": "string", "enum": [ "static", - "dynamic", - "ai" + "dynamic" ], "description": "Defines whether this is a static or dynamic formula." } @@ -1264,8 +1318,7 @@ "type": "string", "enum": [ "static", - "dynamic", - "ai" + "dynamic" ], "description": "Defines whether this is a static or dynamic formula." } @@ -2024,6 +2077,161 @@ "required": [ "data" ] + }, + "view": { + "description": "The view to be created/updated.", + "type": "object", + "required": [ + "name", + "schema" + ], + "properties": { + "name": { + "description": "The name of the view.", + "type": "string" + }, + "primaryDisplay": { + "type": "string", + "description": "A column used to display rows from this view - usually used when rendered in tables." + }, + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + } + } + } + } + } + }, + "viewOutput": { + "type": "object", + "properties": { + "data": { + "description": "The view to be created/updated.", + "type": "object", + "required": [ + "name", + "schema", + "id" + ], + "properties": { + "name": { + "description": "The name of the view.", + "type": "string" + }, + "primaryDisplay": { + "type": "string", + "description": "A column used to display rows from this view - usually used when rendered in tables." + }, + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + } + } + } + }, + "id": { + "description": "The ID of the view.", + "type": "string" + } + } + } + }, + "required": [ + "data" + ] + }, + "viewSearch": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "description": "The view to be created/updated.", + "type": "object", + "required": [ + "name", + "schema", + "id" + ], + "properties": { + "name": { + "description": "The name of the view.", + "type": "string" + }, + "primaryDisplay": { + "type": "string", + "description": "A column used to display rows from this view - usually used when rendered in tables." + }, + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + } + } + } + }, + "id": { + "description": "The ID of the view.", + "type": "string" + } + } + } + } + }, + "required": [ + "data" + ] } } }, @@ -3115,6 +3323,209 @@ } } } + }, + "/views": { + "post": { + "operationId": "viewCreate", + "summary": "Create a view", + "description": "Create a view, this can be against an internal or external table.", + "tags": [ + "views" + ], + "parameters": [ + { + "$ref": "#/components/parameters/appId" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/view" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + }, + "responses": { + "200": { + "description": "Returns the created view, including the ID which has been generated for it.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewOutput" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + } + } + } + }, + "/views/{viewId}": { + "put": { + "operationId": "viewUpdate", + "summary": "Update a view", + "description": "Update a view, this can be against an internal or external table.", + "tags": [ + "views" + ], + "parameters": [ + { + "$ref": "#/components/parameters/viewId" + }, + { + "$ref": "#/components/parameters/appId" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/view" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + }, + "responses": { + "200": { + "description": "Returns the updated view.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewOutput" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + } + } + }, + "delete": { + "operationId": "viewDestroy", + "summary": "Delete a view", + "description": "Delete a view, this can be against an internal or external table.", + "tags": [ + "views" + ], + "parameters": [ + { + "$ref": "#/components/parameters/viewId" + }, + { + "$ref": "#/components/parameters/appId" + } + ], + "responses": { + "200": { + "description": "Returns the deleted view.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewOutput" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + } + } + }, + "get": { + "operationId": "viewGetById", + "summary": "Retrieve a view", + "description": "Lookup a view, this could be internal or external.", + "tags": [ + "views" + ], + "parameters": [ + { + "$ref": "#/components/parameters/viewId" + }, + { + "$ref": "#/components/parameters/appId" + } + ], + "responses": { + "200": { + "description": "Returns the retrieved view.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewOutput" + }, + "examples": { + "view": { + "$ref": "#/components/examples/view" + } + } + } + } + } + } + } + }, + "/views/search": { + "post": { + "operationId": "viewSearch", + "summary": "Search for views", + "description": "Based on view properties (currently only name) search for views.", + "tags": [ + "views" + ], + "parameters": [ + { + "$ref": "#/components/parameters/appId" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewSearch" + } + } + } + }, + "responses": { + "200": { + "description": "Returns the found views, based on the search parameters.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/viewSearch" + }, + "examples": { + "views": { + "$ref": "#/components/examples/views" + } + } + } + } + } + } + } } }, "tags": [] diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 1e9b9921cf..9a2ce6e0da 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -442,6 +442,46 @@ components: # TYPE budibase_quota_limit_automations gauge budibase_quota_limit_automations 9007199254740991 + view: + value: + data: + name: peopleView + tableId: ta_896a325f7e8147d2a2cda93c5d236511 + schema: + name: + visible: true + readonly: false + order: 1 + width: 300 + age: + visible: true + readonly: true + order: 2 + width: 200 + salary: + visible: false + readonly: false + primaryDisplay: name + views: + value: + data: + - name: peopleView + tableId: ta_896a325f7e8147d2a2cda93c5d236511 + schema: + name: + visible: true + readonly: false + order: 1 + width: 300 + age: + visible: true + readonly: true + order: 2 + width: 200 + salary: + visible: false + readonly: false + primaryDisplay: name securitySchemes: ApiKeyAuth: type: apiKey @@ -761,7 +801,6 @@ components: enum: - static - dynamic - - ai description: Defines whether this is a static or dynamic formula. - type: object properties: @@ -931,7 +970,6 @@ components: enum: - static - dynamic - - ai description: Defines whether this is a static or dynamic formula. - type: object properties: @@ -1108,7 +1146,6 @@ components: enum: - static - dynamic - - ai description: Defines whether this is a static or dynamic formula. - type: object properties: @@ -1704,6 +1741,134 @@ components: - userIds required: - data + view: + description: The view to be created/updated. + type: object + required: + - name + - schema + properties: + name: + description: The name of the view. + type: string + primaryDisplay: + type: string + description: A column used to display rows from this view - usually used when + rendered in tables. + schema: + type: object + additionalProperties: + type: object + properties: + visible: + type: boolean + description: Defines whether the column is visible or not - rows + retrieved/updated through this view will not be able to access + it. + readonly: + type: boolean + description: "When used in combination with 'visible: true' the column will be + visible in row responses but cannot be updated." + order: + type: integer + description: A number defining where the column shows up in tables, lowest being + first. + width: + type: integer + description: A width for the column, defined in pixels - this affects rendering + in tables. + viewOutput: + type: object + properties: + data: + description: The view to be created/updated. + type: object + required: + - name + - schema + - id + properties: + name: + description: The name of the view. + type: string + primaryDisplay: + type: string + description: A column used to display rows from this view - usually used when + rendered in tables. + schema: + type: object + additionalProperties: + type: object + properties: + visible: + type: boolean + description: Defines whether the column is visible or not - rows + retrieved/updated through this view will not be able to + access it. + readonly: + type: boolean + description: "When used in combination with 'visible: true' the column will be + visible in row responses but cannot be updated." + order: + type: integer + description: A number defining where the column shows up in tables, lowest being + first. + width: + type: integer + description: A width for the column, defined in pixels - this affects rendering + in tables. + id: + description: The ID of the view. + type: string + required: + - data + viewSearch: + type: object + properties: + data: + type: array + items: + description: The view to be created/updated. + type: object + required: + - name + - schema + - id + properties: + name: + description: The name of the view. + type: string + primaryDisplay: + type: string + description: A column used to display rows from this view - usually used when + rendered in tables. + schema: + type: object + additionalProperties: + type: object + properties: + visible: + type: boolean + description: Defines whether the column is visible or not - rows + retrieved/updated through this view will not be able to + access it. + readonly: + type: boolean + description: "When used in combination with 'visible: true' the column will be + visible in row responses but cannot be updated." + order: + type: integer + description: A number defining where the column shows up in tables, lowest being + first. + width: + type: integer + description: A width for the column, defined in pixels - this affects rendering + in tables. + id: + description: The ID of the view. + type: string + required: + - data security: - ApiKeyAuth: [] paths: @@ -2359,4 +2524,123 @@ paths: examples: users: $ref: "#/components/examples/users" + /views: + post: + operationId: viewCreate + summary: Create a view + description: Create a view, this can be against an internal or external table. + tags: + - views + parameters: + - $ref: "#/components/parameters/appId" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/view" + examples: + view: + $ref: "#/components/examples/view" + responses: + "200": + description: Returns the created view, including the ID which has been generated + for it. + content: + application/json: + schema: + $ref: "#/components/schemas/viewOutput" + examples: + view: + $ref: "#/components/examples/view" + "/views/{viewId}": + put: + operationId: viewUpdate + summary: Update a view + description: Update a view, this can be against an internal or external table. + tags: + - views + parameters: + - $ref: "#/components/parameters/viewId" + - $ref: "#/components/parameters/appId" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/view" + examples: + view: + $ref: "#/components/examples/view" + responses: + "200": + description: Returns the updated view. + content: + application/json: + schema: + $ref: "#/components/schemas/viewOutput" + examples: + view: + $ref: "#/components/examples/view" + delete: + operationId: viewDestroy + summary: Delete a view + description: Delete a view, this can be against an internal or external table. + tags: + - views + parameters: + - $ref: "#/components/parameters/viewId" + - $ref: "#/components/parameters/appId" + responses: + "200": + description: Returns the deleted view. + content: + application/json: + schema: + $ref: "#/components/schemas/viewOutput" + examples: + view: + $ref: "#/components/examples/view" + get: + operationId: viewGetById + summary: Retrieve a view + description: Lookup a view, this could be internal or external. + tags: + - views + parameters: + - $ref: "#/components/parameters/viewId" + - $ref: "#/components/parameters/appId" + responses: + "200": + description: Returns the retrieved view. + content: + application/json: + schema: + $ref: "#/components/schemas/viewOutput" + examples: + view: + $ref: "#/components/examples/view" + /views/search: + post: + operationId: viewSearch + summary: Search for views + description: Based on view properties (currently only name) search for views. + tags: + - views + parameters: + - $ref: "#/components/parameters/appId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/viewSearch" + responses: + "200": + description: Returns the found views, based on the search parameters. + content: + application/json: + schema: + $ref: "#/components/schemas/viewSearch" + examples: + views: + $ref: "#/components/examples/views" tags: [] diff --git a/packages/server/specs/resources/index.ts b/packages/server/specs/resources/index.ts index c06148b7de..0d32f2a007 100644 --- a/packages/server/specs/resources/index.ts +++ b/packages/server/specs/resources/index.ts @@ -6,6 +6,7 @@ import user from "./user" import metrics from "./metrics" import misc from "./misc" import roles from "./roles" +import view from "./view" export const examples = { ...application.getExamples(), @@ -15,6 +16,8 @@ export const examples = { ...user.getExamples(), ...misc.getExamples(), ...metrics.getExamples(), + ...roles.getExamples(), + ...view.getExamples(), } export const schemas = { @@ -25,4 +28,5 @@ export const schemas = { ...user.getSchemas(), ...misc.getSchemas(), ...roles.getSchemas(), + ...view.getSchemas(), } diff --git a/packages/server/src/api/controllers/public/mapping/index.ts b/packages/server/src/api/controllers/public/mapping/index.ts index 0cdcfbbe4b..b765f4bd76 100644 --- a/packages/server/src/api/controllers/public/mapping/index.ts +++ b/packages/server/src/api/controllers/public/mapping/index.ts @@ -3,6 +3,7 @@ import applications from "./applications" import users from "./users" import rows from "./rows" import queries from "./queries" +import views from "./views" export default { ...tables, @@ -10,4 +11,5 @@ export default { ...users, ...rows, ...queries, + ...views, } diff --git a/packages/server/src/api/controllers/public/mapping/types.ts b/packages/server/src/api/controllers/public/mapping/types.ts index 9fea9b7213..6cbcfddb92 100644 --- a/packages/server/src/api/controllers/public/mapping/types.ts +++ b/packages/server/src/api/controllers/public/mapping/types.ts @@ -9,6 +9,9 @@ export type CreateApplicationParams = components["schemas"]["application"] export type Table = components["schemas"]["tableOutput"]["data"] export type CreateTableParams = components["schemas"]["table"] +export type View = components["schemas"]["viewOutput"]["data"] +export type CreateViewParams = components["schemas"]["view"] + export type Row = components["schemas"]["rowOutput"]["data"] export type RowSearch = components["schemas"]["searchOutput"] export type CreateRowParams = components["schemas"]["row"] diff --git a/packages/server/src/api/controllers/public/mapping/views.ts b/packages/server/src/api/controllers/public/mapping/views.ts new file mode 100644 index 0000000000..b6cf64542e --- /dev/null +++ b/packages/server/src/api/controllers/public/mapping/views.ts @@ -0,0 +1,26 @@ +import { View } from "./types" + +function view(body: any): View { + return { + id: body.id, + name: body.name, + schema: body.schema, + primaryDisplay: body.primaryDisplay, + } +} + +function mapView(ctx: any): { data: View } { + return { + data: view(ctx.body), + } +} + +function mapViews(ctx: any): { data: View[] } { + const tables = ctx.body.map((body: any) => view(body)) + return { data: tables } +} + +export default { + mapView, + mapViews, +} diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts index 85979126ff..f3265aa70d 100644 --- a/packages/server/src/api/controllers/public/views.ts +++ b/packages/server/src/api/controllers/public/views.ts @@ -18,8 +18,7 @@ function fixView(view: ViewV2, params?: { viewId: string }) { export async function search(ctx: UserCtx, next: Next) { const { name } = ctx.request.body - // TODO: need a view search endpoint - // await controller.v2.fetch(ctx) + await controller.v2.fetch(ctx) ctx.body = stringSearch(ctx.body, name) await next() } diff --git a/packages/server/src/api/routes/public/middleware/mapper.ts b/packages/server/src/api/routes/public/middleware/mapper.ts index 03feb6cc5c..9d0cae5d61 100644 --- a/packages/server/src/api/routes/public/middleware/mapper.ts +++ b/packages/server/src/api/routes/public/middleware/mapper.ts @@ -1,9 +1,10 @@ import { Ctx } from "@budibase/types" import mapping from "../../../controllers/public/mapping" -enum Resources { +enum Resource { APPLICATION = "applications", TABLES = "tables", + VIEWS = "views", ROWS = "rows", USERS = "users", QUERIES = "queries", @@ -15,7 +16,7 @@ function isAttachment(ctx: Ctx) { } function isArrayResponse(ctx: Ctx) { - return ctx.url.endsWith(Resources.SEARCH) || Array.isArray(ctx.body) + return ctx.url.endsWith(Resource.SEARCH) || Array.isArray(ctx.body) } function noResponse(ctx: Ctx) { @@ -38,6 +39,14 @@ function processTables(ctx: Ctx) { } } +function processViews(ctx: Ctx) { + if (isArrayResponse(ctx)) { + return mapping.mapViews(ctx) + } else { + return mapping.mapView(ctx) + } +} + function processRows(ctx: Ctx) { if (isArrayResponse(ctx)) { return mapping.mapRowSearch(ctx) @@ -71,20 +80,27 @@ export default async (ctx: Ctx, next: any) => { let body = {} switch (urlParts[0]) { - case Resources.APPLICATION: + case Resource.APPLICATION: body = processApplications(ctx) break - case Resources.TABLES: - if (urlParts[2] === Resources.ROWS) { + case Resource.TABLES: + if (urlParts[2] === Resource.ROWS) { body = processRows(ctx) } else { body = processTables(ctx) } break - case Resources.USERS: + case Resource.VIEWS: + if (urlParts[2] === Resource.ROWS) { + body = processRows(ctx) + } else { + body = processViews(ctx) + } + break + case Resource.USERS: body = processUsers(ctx) break - case Resources.QUERIES: + case Resource.QUERIES: body = processQueries(ctx) break } diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 4479c89d9d..9d68e11185 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -93,6 +93,22 @@ export interface paths { /** Based on user properties (currently only name) search for users. */ post: operations["userSearch"]; }; + "/views": { + /** Create a view, this can be against an internal or external table. */ + post: operations["viewCreate"]; + }; + "/views/{viewId}": { + /** Lookup a view, this could be internal or external. */ + get: operations["viewGetById"]; + /** Update a view, this can be against an internal or external table. */ + put: operations["viewUpdate"]; + /** Delete a view, this can be against an internal or external table. */ + delete: operations["viewDestroy"]; + }; + "/views/search": { + /** Based on view properties (currently only name) search for views. */ + post: operations["viewSearch"]; + }; } export interface components { @@ -813,6 +829,70 @@ export interface components { userIds: string[]; }; }; + /** @description The view to be created/updated. */ + view: { + /** @description The name of the view. */ + name: string; + /** @description A column used to display rows from this view - usually used when rendered in tables. */ + primaryDisplay?: string; + schema: { + [key: string]: { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + }; + }; + }; + viewOutput: { + /** @description The view to be created/updated. */ + data: { + /** @description The name of the view. */ + name: string; + /** @description A column used to display rows from this view - usually used when rendered in tables. */ + primaryDisplay?: string; + schema: { + [key: string]: { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + }; + }; + /** @description The ID of the view. */ + id: string; + }; + }; + viewSearch: { + data: { + /** @description The name of the view. */ + name: string; + /** @description A column used to display rows from this view - usually used when rendered in tables. */ + primaryDisplay?: string; + schema: { + [key: string]: { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + }; + }; + /** @description The ID of the view. */ + id: string; + }[]; + }; }; parameters: { /** @description The ID of the table which this request is targeting. */ @@ -1409,6 +1489,106 @@ export interface operations { }; }; }; + /** Create a view, this can be against an internal or external table. */ + viewCreate: { + parameters: { + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** Returns the created view, including the ID which has been generated for it. */ + 200: { + content: { + "application/json": components["schemas"]["viewOutput"]; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["view"]; + }; + }; + }; + /** Lookup a view, this could be internal or external. */ + viewGetById: { + parameters: { + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** Returns the retrieved view. */ + 200: { + content: { + "application/json": components["schemas"]["viewOutput"]; + }; + }; + }; + }; + /** Update a view, this can be against an internal or external table. */ + viewUpdate: { + parameters: { + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** Returns the updated view. */ + 200: { + content: { + "application/json": components["schemas"]["viewOutput"]; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["view"]; + }; + }; + }; + /** Delete a view, this can be against an internal or external table. */ + viewDestroy: { + parameters: { + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** Returns the deleted view. */ + 200: { + content: { + "application/json": components["schemas"]["viewOutput"]; + }; + }; + }; + }; + /** Based on view properties (currently only name) search for views. */ + viewSearch: { + parameters: { + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** Returns the found views, based on the search parameters. */ + 200: { + content: { + "application/json": components["schemas"]["viewSearch"]; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["viewSearch"]; + }; + }; + }; } export interface external {} From 38760897bf4a5579a1df4e5ca1e1b8bae7ed5916 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 30 Oct 2024 14:40:13 +0000 Subject: [PATCH 022/193] Updating specs again. --- packages/server/specs/openapi.json | 9 +++++++++ packages/server/specs/openapi.yaml | 7 +++++++ packages/server/specs/parameters.ts | 10 ++++++++++ packages/server/src/definitions/openapi.ts | 14 ++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index c80f947fad..3af15e1f02 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -32,6 +32,15 @@ "type": "string" } }, + "viewId": { + "in": "path", + "name": "viewId", + "required": true, + "description": "The ID of the view which this request is targeting.", + "schema": { + "type": "string" + } + }, "rowId": { "in": "path", "name": "rowId", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 9a2ce6e0da..094957805f 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -23,6 +23,13 @@ components: description: The ID of the table which this request is targeting. schema: type: string + viewId: + in: path + name: viewId + required: true + description: The ID of the view which this request is targeting. + schema: + type: string rowId: in: path name: rowId diff --git a/packages/server/specs/parameters.ts b/packages/server/specs/parameters.ts index a4f2b27ae4..abc26c48d6 100644 --- a/packages/server/specs/parameters.ts +++ b/packages/server/specs/parameters.ts @@ -8,6 +8,16 @@ export const tableId = { }, } +export const viewId = { + in: "path", + name: "viewId", + required: true, + description: "The ID of the view which this request is targeting.", + schema: { + type: "string", + }, +} + export const rowId = { in: "path", name: "rowId", diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 9d68e11185..a1e5d8202f 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -897,6 +897,8 @@ export interface components { parameters: { /** @description The ID of the table which this request is targeting. */ tableId: string; + /** @description The ID of the view which this request is targeting. */ + viewId: string; /** @description The ID of the row which this request is targeting. */ rowId: string; /** @description The ID of the app which this request is targeting. */ @@ -1514,6 +1516,10 @@ export interface operations { /** Lookup a view, this could be internal or external. */ viewGetById: { parameters: { + path: { + /** The ID of the view which this request is targeting. */ + viewId: components["parameters"]["viewId"]; + }; header: { /** The ID of the app which this request is targeting. */ "x-budibase-app-id": components["parameters"]["appId"]; @@ -1531,6 +1537,10 @@ export interface operations { /** Update a view, this can be against an internal or external table. */ viewUpdate: { parameters: { + path: { + /** The ID of the view which this request is targeting. */ + viewId: components["parameters"]["viewId"]; + }; header: { /** The ID of the app which this request is targeting. */ "x-budibase-app-id": components["parameters"]["appId"]; @@ -1553,6 +1563,10 @@ export interface operations { /** Delete a view, this can be against an internal or external table. */ viewDestroy: { parameters: { + path: { + /** The ID of the view which this request is targeting. */ + viewId: components["parameters"]["viewId"]; + }; header: { /** The ID of the app which this request is targeting. */ "x-budibase-app-id": components["parameters"]["appId"]; From 037a33dc7d6e8ce34e009d13cd0a8b0ca63211d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 30 Oct 2024 17:12:48 +0000 Subject: [PATCH 023/193] Updating specs to correct some issues with app ID. --- packages/server/specs/openapi.json | 6 +++--- packages/server/specs/openapi.yaml | 6 +++--- packages/server/specs/parameters.ts | 4 ++-- packages/server/src/api/controllers/public/views.ts | 9 +++++++-- packages/server/src/api/routes/public/index.ts | 10 +++++++--- packages/server/src/api/routes/public/views.ts | 2 +- packages/server/src/definitions/openapi.ts | 2 +- packages/server/src/middleware/publicApi.ts | 4 ++-- 8 files changed, 26 insertions(+), 17 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 3af15e1f02..1864dd8252 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -56,7 +56,7 @@ "required": true, "description": "The ID of the app which this request is targeting.", "schema": { - "default": "{{ appId }}", + "default": "{{appId}}", "type": "string" } }, @@ -66,7 +66,7 @@ "required": true, "description": "The ID of the app which this request is targeting.", "schema": { - "default": "{{ appId }}", + "default": "{{appId}}", "type": "string" } }, @@ -3512,7 +3512,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/viewSearch" + "$ref": "#/components/schemas/nameSearch" } } } diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 094957805f..cc10882b77 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -43,7 +43,7 @@ components: required: true description: The ID of the app which this request is targeting. schema: - default: "{{ appId }}" + default: "{{appId}}" type: string appIdUrl: in: path @@ -51,7 +51,7 @@ components: required: true description: The ID of the app which this request is targeting. schema: - default: "{{ appId }}" + default: "{{appId}}" type: string queryId: in: path @@ -2639,7 +2639,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/viewSearch" + $ref: "#/components/schemas/nameSearch" responses: "200": description: Returns the found views, based on the search parameters. diff --git a/packages/server/specs/parameters.ts b/packages/server/specs/parameters.ts index abc26c48d6..b3fb274567 100644 --- a/packages/server/specs/parameters.ts +++ b/packages/server/specs/parameters.ts @@ -34,7 +34,7 @@ export const appId = { required: true, description: "The ID of the app which this request is targeting.", schema: { - default: "{{ appId }}", + default: "{{appId}}", type: "string", }, } @@ -45,7 +45,7 @@ export const appIdUrl = { required: true, description: "The ID of the app which this request is targeting.", schema: { - default: "{{ appId }}", + default: "{{appId}}", type: "string", }, } diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts index f3265aa70d..d43f1592b1 100644 --- a/packages/server/src/api/controllers/public/views.ts +++ b/packages/server/src/api/controllers/public/views.ts @@ -19,7 +19,7 @@ function fixView(view: ViewV2, params?: { viewId: string }) { export async function search(ctx: UserCtx, next: Next) { const { name } = ctx.request.body await controller.v2.fetch(ctx) - ctx.body = stringSearch(ctx.body, name) + ctx.body = stringSearch(ctx.body.data, name) await next() } @@ -30,7 +30,12 @@ export async function create(ctx: UserCtx, next: Next) { } export async function read(ctx: UserCtx, next: Next) { - await controller.v2.get(ctx) + await controller.v2.get({ + ...ctx, + params: { + viewId: ctx.params.viewId, + }, + }) await next() } diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts index 36e0f74bee..bcec8f3166 100644 --- a/packages/server/src/api/routes/public/index.ts +++ b/packages/server/src/api/routes/public/index.ts @@ -4,18 +4,20 @@ import queryEndpoints from "./queries" import tableEndpoints from "./tables" import rowEndpoints from "./rows" import userEndpoints from "./users" +import viewEndpoints from "./views" +import roleEndpoints from "./roles" import authorized from "../../../middleware/authorized" import publicApi from "../../../middleware/publicApi" import { paramResource, paramSubResource } from "../../../middleware/resourceId" -import { PermissionType, PermissionLevel } from "@budibase/types" +import { PermissionLevel, PermissionType } from "@budibase/types" import { CtxFn } from "./utils/Endpoint" import mapperMiddleware from "./middleware/mapper" import env from "../../../environment" +import { middleware, redis } from "@budibase/backend-core" +import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils" // below imports don't have declaration files const Router = require("@koa/router") const { RateLimit, Stores } = require("koa2-ratelimit") -import { middleware, redis } from "@budibase/backend-core" -import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils" interface KoaRateLimitOptions { socket: { @@ -145,8 +147,10 @@ function applyRoutes( } applyAdminRoutes(metricEndpoints) +applyAdminRoutes(roleEndpoints) applyRoutes(appEndpoints, PermissionType.APP, "appId") applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId") +applyRoutes(viewEndpoints, PermissionType.VIEW, "viewId") applyRoutes(userEndpoints, PermissionType.USER, "userId") applyRoutes(queryEndpoints, PermissionType.QUERY, "queryId") // needs to be applied last for routing purposes, don't override other endpoints diff --git a/packages/server/src/api/routes/public/views.ts b/packages/server/src/api/routes/public/views.ts index 7c182d105f..139d79be1f 100644 --- a/packages/server/src/api/routes/public/views.ts +++ b/packages/server/src/api/routes/public/views.ts @@ -144,7 +144,7 @@ read.push(new Endpoint("get", "/views/:viewId", controller.read)) * content: * application/json: * schema: - * $ref: '#/components/schemas/viewSearch' + * $ref: '#/components/schemas/nameSearch' * responses: * 200: * description: Returns the found views, based on the search parameters. diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index a1e5d8202f..4d6ab6a06c 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -1599,7 +1599,7 @@ export interface operations { }; requestBody: { content: { - "application/json": components["schemas"]["viewSearch"]; + "application/json": components["schemas"]["nameSearch"]; }; }; }; diff --git a/packages/server/src/middleware/publicApi.ts b/packages/server/src/middleware/publicApi.ts index e3897d1a49..da10dd3539 100644 --- a/packages/server/src/middleware/publicApi.ts +++ b/packages/server/src/middleware/publicApi.ts @@ -1,8 +1,8 @@ import { constants, utils } from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { Ctx } from "@budibase/types" export default function ({ requiresAppId }: { requiresAppId?: boolean } = {}) { - return async (ctx: BBContext, next: any) => { + return async (ctx: Ctx, next: any) => { const appId = await utils.getAppIdFromCtx(ctx) if (requiresAppId && !appId) { ctx.throw( From 9d8566189387a954b9986ec6627b7ec0b6ae0e60 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 31 Oct 2024 17:42:46 +0000 Subject: [PATCH 024/193] Finishing specs to explain how to configure a calculation view. --- packages/server/specs/openapi.json | 287 ++++++++++++++---- packages/server/specs/openapi.yaml | 254 ++++++++++++---- packages/server/specs/resources/view.ts | 48 ++- .../server/src/api/controllers/public/rows.ts | 20 +- .../src/api/controllers/public/views.ts | 16 +- packages/server/src/api/routes/public/rows.ts | 36 +++ packages/server/src/definitions/openapi.ts | 151 +++++++-- 7 files changed, 661 insertions(+), 151 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index 1864dd8252..d45a8f28c9 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -2099,6 +2099,13 @@ "description": "The name of the view.", "type": "string" }, + "type": { + "description": "The type of view - standard (empty value) or calculation.", + "type": "string", + "enum": [ + "calculation" + ] + }, "primaryDisplay": { "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." @@ -2106,25 +2113,65 @@ "schema": { "type": "object", "additionalProperties": { - "type": "object", - "properties": { - "visible": { - "type": "boolean", - "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + "oneOf": [ + { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + }, + "column": { + "type": "array", + "description": "If this is a relationship column, we can set the columns we wish to include", + "items": { + "type": "object", + "properties": { + "readonly": { + "type": "boolean" + } + } + } + } + } }, - "readonly": { - "type": "boolean", - "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." - }, - "order": { - "type": "integer", - "description": "A number defining where the column shows up in tables, lowest being first." - }, - "width": { - "type": "integer", - "description": "A width for the column, defined in pixels - this affects rendering in tables." + { + "type": "object", + "properties": { + "calculationType": { + "type": "string", + "description": "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.", + "enum": [ + "sum", + "avg", + "count", + "min", + "max" + ] + }, + "field": { + "type": "string", + "description": "The field from the table to perform the calculation on." + }, + "distinct": { + "type": "boolean", + "description": "Can be used in tandem with the count calculation type, to count unique entries." + } + } } - } + ] } } } @@ -2145,6 +2192,13 @@ "description": "The name of the view.", "type": "string" }, + "type": { + "description": "The type of view - standard (empty value) or calculation.", + "type": "string", + "enum": [ + "calculation" + ] + }, "primaryDisplay": { "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." @@ -2152,25 +2206,65 @@ "schema": { "type": "object", "additionalProperties": { - "type": "object", - "properties": { - "visible": { - "type": "boolean", - "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + "oneOf": [ + { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + }, + "column": { + "type": "array", + "description": "If this is a relationship column, we can set the columns we wish to include", + "items": { + "type": "object", + "properties": { + "readonly": { + "type": "boolean" + } + } + } + } + } }, - "readonly": { - "type": "boolean", - "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." - }, - "order": { - "type": "integer", - "description": "A number defining where the column shows up in tables, lowest being first." - }, - "width": { - "type": "integer", - "description": "A width for the column, defined in pixels - this affects rendering in tables." + { + "type": "object", + "properties": { + "calculationType": { + "type": "string", + "description": "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.", + "enum": [ + "sum", + "avg", + "count", + "min", + "max" + ] + }, + "field": { + "type": "string", + "description": "The field from the table to perform the calculation on." + }, + "distinct": { + "type": "boolean", + "description": "Can be used in tandem with the count calculation type, to count unique entries." + } + } } - } + ] } }, "id": { @@ -2202,6 +2296,13 @@ "description": "The name of the view.", "type": "string" }, + "type": { + "description": "The type of view - standard (empty value) or calculation.", + "type": "string", + "enum": [ + "calculation" + ] + }, "primaryDisplay": { "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." @@ -2209,25 +2310,65 @@ "schema": { "type": "object", "additionalProperties": { - "type": "object", - "properties": { - "visible": { - "type": "boolean", - "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + "oneOf": [ + { + "type": "object", + "properties": { + "visible": { + "type": "boolean", + "description": "Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it." + }, + "readonly": { + "type": "boolean", + "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." + }, + "order": { + "type": "integer", + "description": "A number defining where the column shows up in tables, lowest being first." + }, + "width": { + "type": "integer", + "description": "A width for the column, defined in pixels - this affects rendering in tables." + }, + "column": { + "type": "array", + "description": "If this is a relationship column, we can set the columns we wish to include", + "items": { + "type": "object", + "properties": { + "readonly": { + "type": "boolean" + } + } + } + } + } }, - "readonly": { - "type": "boolean", - "description": "When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated." - }, - "order": { - "type": "integer", - "description": "A number defining where the column shows up in tables, lowest being first." - }, - "width": { - "type": "integer", - "description": "A width for the column, defined in pixels - this affects rendering in tables." + { + "type": "object", + "properties": { + "calculationType": { + "type": "string", + "description": "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.", + "enum": [ + "sum", + "avg", + "count", + "min", + "max" + ] + }, + "field": { + "type": "string", + "description": "The field from the table to perform the calculation on." + }, + "distinct": { + "type": "boolean", + "description": "Can be used in tandem with the count calculation type, to count unique entries." + } + } } - } + ] } }, "id": { @@ -2958,6 +3099,50 @@ } } }, + "/views/{viewId}/rows/search": { + "post": { + "operationId": "rowViewSearch", + "summary": "Search for rows in a view", + "tags": [ + "rows" + ], + "parameters": [ + { + "$ref": "#/components/parameters/viewId" + }, + { + "$ref": "#/components/parameters/appId" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/rowSearch" + } + } + } + }, + "responses": { + "200": { + "description": "The response will contain an array of rows that match the search parameters.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/searchOutput" + }, + "examples": { + "search": { + "$ref": "#/components/examples/rows" + } + } + } + } + } + } + } + }, "/tables": { "post": { "operationId": "tableCreate", diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index cc10882b77..0e551207ed 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -1758,6 +1758,11 @@ components: name: description: The name of the view. type: string + type: + description: The type of view - standard (empty value) or calculation. + type: string + enum: + - calculation primaryDisplay: type: string description: A column used to display rows from this view - usually used when @@ -1765,47 +1770,8 @@ components: schema: type: object additionalProperties: - type: object - properties: - visible: - type: boolean - description: Defines whether the column is visible or not - rows - retrieved/updated through this view will not be able to access - it. - readonly: - type: boolean - description: "When used in combination with 'visible: true' the column will be - visible in row responses but cannot be updated." - order: - type: integer - description: A number defining where the column shows up in tables, lowest being - first. - width: - type: integer - description: A width for the column, defined in pixels - this affects rendering - in tables. - viewOutput: - type: object - properties: - data: - description: The view to be created/updated. - type: object - required: - - name - - schema - - id - properties: - name: - description: The name of the view. - type: string - primaryDisplay: - type: string - description: A column used to display rows from this view - usually used when - rendered in tables. - schema: - type: object - additionalProperties: - type: object + oneOf: + - type: object properties: visible: type: boolean @@ -1824,6 +1790,112 @@ components: type: integer description: A width for the column, defined in pixels - this affects rendering in tables. + column: + type: array + description: If this is a relationship column, we can set the columns we wish to + include + items: + type: object + properties: + readonly: + type: boolean + - type: object + properties: + calculationType: + type: string + description: This column should be built from a calculation, specifying a type + and field. It is important to note when a calculation is + configured all non-calculation columns will be used for + grouping. + enum: + - sum + - avg + - count + - min + - max + field: + type: string + description: The field from the table to perform the calculation on. + distinct: + type: boolean + description: Can be used in tandem with the count calculation type, to count + unique entries. + viewOutput: + type: object + properties: + data: + description: The view to be created/updated. + type: object + required: + - name + - schema + - id + properties: + name: + description: The name of the view. + type: string + type: + description: The type of view - standard (empty value) or calculation. + type: string + enum: + - calculation + primaryDisplay: + type: string + description: A column used to display rows from this view - usually used when + rendered in tables. + schema: + type: object + additionalProperties: + oneOf: + - type: object + properties: + visible: + type: boolean + description: Defines whether the column is visible or not - rows + retrieved/updated through this view will not be able + to access it. + readonly: + type: boolean + description: "When used in combination with 'visible: true' the column will be + visible in row responses but cannot be updated." + order: + type: integer + description: A number defining where the column shows up in tables, lowest being + first. + width: + type: integer + description: A width for the column, defined in pixels - this affects rendering + in tables. + column: + type: array + description: If this is a relationship column, we can set the columns we wish to + include + items: + type: object + properties: + readonly: + type: boolean + - type: object + properties: + calculationType: + type: string + description: This column should be built from a calculation, specifying a type + and field. It is important to note when a calculation + is configured all non-calculation columns will be used + for grouping. + enum: + - sum + - avg + - count + - min + - max + field: + type: string + description: The field from the table to perform the calculation on. + distinct: + type: boolean + description: Can be used in tandem with the count calculation type, to count + unique entries. id: description: The ID of the view. type: string @@ -1845,6 +1917,11 @@ components: name: description: The name of the view. type: string + type: + description: The type of view - standard (empty value) or calculation. + type: string + enum: + - calculation primaryDisplay: type: string description: A column used to display rows from this view - usually used when @@ -1852,25 +1929,56 @@ components: schema: type: object additionalProperties: - type: object - properties: - visible: - type: boolean - description: Defines whether the column is visible or not - rows - retrieved/updated through this view will not be able to - access it. - readonly: - type: boolean - description: "When used in combination with 'visible: true' the column will be - visible in row responses but cannot be updated." - order: - type: integer - description: A number defining where the column shows up in tables, lowest being - first. - width: - type: integer - description: A width for the column, defined in pixels - this affects rendering - in tables. + oneOf: + - type: object + properties: + visible: + type: boolean + description: Defines whether the column is visible or not - rows + retrieved/updated through this view will not be able + to access it. + readonly: + type: boolean + description: "When used in combination with 'visible: true' the column will be + visible in row responses but cannot be updated." + order: + type: integer + description: A number defining where the column shows up in tables, lowest being + first. + width: + type: integer + description: A width for the column, defined in pixels - this affects rendering + in tables. + column: + type: array + description: If this is a relationship column, we can set the columns we wish to + include + items: + type: object + properties: + readonly: + type: boolean + - type: object + properties: + calculationType: + type: string + description: This column should be built from a calculation, specifying a type + and field. It is important to note when a + calculation is configured all non-calculation + columns will be used for grouping. + enum: + - sum + - avg + - count + - min + - max + field: + type: string + description: The field from the table to perform the calculation on. + distinct: + type: boolean + description: Can be used in tandem with the count calculation type, to count + unique entries. id: description: The ID of the view. type: string @@ -2308,6 +2416,32 @@ paths: examples: search: $ref: "#/components/examples/rows" + "/views/{viewId}/rows/search": + post: + operationId: rowViewSearch + summary: Search for rows in a view + tags: + - rows + parameters: + - $ref: "#/components/parameters/viewId" + - $ref: "#/components/parameters/appId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/rowSearch" + responses: + "200": + description: The response will contain an array of rows that match the search + parameters. + content: + application/json: + schema: + $ref: "#/components/schemas/searchOutput" + examples: + search: + $ref: "#/components/examples/rows" /tables: post: operationId: tableCreate diff --git a/packages/server/specs/resources/view.ts b/packages/server/specs/resources/view.ts index 5afdf78a0e..e798883ced 100644 --- a/packages/server/specs/resources/view.ts +++ b/packages/server/specs/resources/view.ts @@ -1,5 +1,6 @@ import { object } from "./utils" import Resource from "./utils/Resource" +import { CalculationType } from "@budibase/types" const view = { name: "peopleView", @@ -46,6 +47,19 @@ const baseColumnDef = { description: "A width for the column, defined in pixels - this affects rendering in tables.", }, + column: { + type: "array", + description: + "If this is a relationship column, we can set the columns we wish to include", + items: { + type: "object", + properties: { + readonly: { + type: "boolean", + }, + }, + }, + }, } const viewSchema = { @@ -57,6 +71,11 @@ const viewSchema = { description: "The name of the view.", type: "string", }, + type: { + description: "The type of view - standard (empty value) or calculation.", + type: "string", + enum: ["calculation"], + }, primaryDisplay: { type: "string", description: @@ -65,8 +84,33 @@ const viewSchema = { schema: { type: "object", additionalProperties: { - type: "object", - properties: baseColumnDef, + oneOf: [ + { + type: "object", + properties: baseColumnDef, + }, + { + type: "object", + properties: { + calculationType: { + type: "string", + description: + "This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping.", + enum: Object.values(CalculationType), + }, + field: { + type: "string", + description: + "The field from the table to perform the calculation on.", + }, + distinct: { + type: "boolean", + description: + "Can be used in tandem with the count calculation type, to count unique entries.", + }, + }, + }, + ], }, }, }, diff --git a/packages/server/src/api/controllers/public/rows.ts b/packages/server/src/api/controllers/public/rows.ts index 16403b06c9..634a41ed85 100644 --- a/packages/server/src/api/controllers/public/rows.ts +++ b/packages/server/src/api/controllers/public/rows.ts @@ -22,13 +22,13 @@ export function fixRow(row: Row, params: any) { return row } -export async function search(ctx: UserCtx, next: Next) { +function getSearchParameters(ctx: UserCtx) { let { sort, paginate, bookmark, limit, query } = ctx.request.body // update the body to the correct format of the internal search if (!sort) { sort = {} } - ctx.request.body = { + return { sort: sort.column, sortType: sort.type, sortOrder: sort.order, @@ -37,10 +37,25 @@ export async function search(ctx: UserCtx, next: Next) { limit, query, } +} + +export async function search(ctx: UserCtx, next: Next) { + ctx.request.body = getSearchParameters(ctx) await rowController.search(ctx) await next() } +export async function viewSearch(ctx: UserCtx, next: Next) { + ctx.request.body = getSearchParameters(ctx) + await rowController.views.searchView({ + ...ctx, + params: { + viewId: ctx.params.viewId, + }, + }) + await next() +} + export async function create(ctx: UserCtx, next: Next) { ctx.request.body = fixRow(ctx.request.body, ctx.params) await rowController.save(ctx) @@ -79,4 +94,5 @@ export default { update, destroy, search, + viewSearch, } diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts index d43f1592b1..e400e850ea 100644 --- a/packages/server/src/api/controllers/public/views.ts +++ b/packages/server/src/api/controllers/public/views.ts @@ -40,12 +40,16 @@ export async function read(ctx: UserCtx, next: Next) { } export async function update(ctx: UserCtx, next: Next) { - // TODO: this is more complex - no rev on views - // ctx.request.body = await addRev( - // fixView(ctx.request.body, ctx.params), - // ctx.params.tableId - // ) - await controller.v2.update(ctx) + const viewId = ctx.params.viewId + await controller.v2.update({ + ...ctx, + body: { + data: fixView(ctx.body, { viewId }), + }, + params: { + viewId, + }, + }) await next() } diff --git a/packages/server/src/api/routes/public/rows.ts b/packages/server/src/api/routes/public/rows.ts index 80ad6d6434..2085c5cf0f 100644 --- a/packages/server/src/api/routes/public/rows.ts +++ b/packages/server/src/api/routes/public/rows.ts @@ -168,4 +168,40 @@ read.push( ).addMiddleware(externalSearchValidator()) ) +/** + * @openapi + * /views/{viewId}/rows/search: + * post: + * operationId: rowViewSearch + * summary: Search for rows in a view + * tags: + * - rows + * parameters: + * - $ref: '#/components/parameters/viewId' + * - $ref: '#/components/parameters/appId' + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/rowSearch' + * responses: + * 200: + * description: The response will contain an array of rows that match the search parameters. + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/searchOutput' + * examples: + * search: + * $ref: '#/components/examples/rows' + */ +read.push( + new Endpoint( + "post", + "/views/:viewId/rows/search", + controller.search + ).addMiddleware(externalSearchValidator()) +) + export default { read, write } diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index 4d6ab6a06c..fed19ec5cf 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -65,6 +65,9 @@ export interface paths { "/tables/{tableId}/rows/search": { post: operations["rowSearch"]; }; + "/views/{viewId}/rows/search": { + post: operations["rowViewSearch"]; + }; "/tables": { /** Create a table, this could be internal or external. */ post: operations["tableCreate"]; @@ -833,19 +836,40 @@ export interface components { view: { /** @description The name of the view. */ name: string; + /** + * @description The type of view - standard (empty value) or calculation. + * @enum {string} + */ + type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; schema: { - [key: string]: { - /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ - visible?: boolean; - /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ - readonly?: boolean; - /** @description A number defining where the column shows up in tables, lowest being first. */ - order?: number; - /** @description A width for the column, defined in pixels - this affects rendering in tables. */ - width?: number; - }; + [key: string]: + | { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + /** @description If this is a relationship column, we can set the columns we wish to include */ + column?: { + readonly?: boolean; + }[]; + } + | { + /** + * @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping. + * @enum {string} + */ + calculationType?: "sum" | "avg" | "count" | "min" | "max"; + /** @description The field from the table to perform the calculation on. */ + field?: string; + /** @description Can be used in tandem with the count calculation type, to count unique entries. */ + distinct?: boolean; + }; }; }; viewOutput: { @@ -853,19 +877,40 @@ export interface components { data: { /** @description The name of the view. */ name: string; + /** + * @description The type of view - standard (empty value) or calculation. + * @enum {string} + */ + type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; schema: { - [key: string]: { - /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ - visible?: boolean; - /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ - readonly?: boolean; - /** @description A number defining where the column shows up in tables, lowest being first. */ - order?: number; - /** @description A width for the column, defined in pixels - this affects rendering in tables. */ - width?: number; - }; + [key: string]: + | { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + /** @description If this is a relationship column, we can set the columns we wish to include */ + column?: { + readonly?: boolean; + }[]; + } + | { + /** + * @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping. + * @enum {string} + */ + calculationType?: "sum" | "avg" | "count" | "min" | "max"; + /** @description The field from the table to perform the calculation on. */ + field?: string; + /** @description Can be used in tandem with the count calculation type, to count unique entries. */ + distinct?: boolean; + }; }; /** @description The ID of the view. */ id: string; @@ -875,19 +920,40 @@ export interface components { data: { /** @description The name of the view. */ name: string; + /** + * @description The type of view - standard (empty value) or calculation. + * @enum {string} + */ + type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; schema: { - [key: string]: { - /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ - visible?: boolean; - /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ - readonly?: boolean; - /** @description A number defining where the column shows up in tables, lowest being first. */ - order?: number; - /** @description A width for the column, defined in pixels - this affects rendering in tables. */ - width?: number; - }; + [key: string]: + | { + /** @description Defines whether the column is visible or not - rows retrieved/updated through this view will not be able to access it. */ + visible?: boolean; + /** @description When used in combination with 'visible: true' the column will be visible in row responses but cannot be updated. */ + readonly?: boolean; + /** @description A number defining where the column shows up in tables, lowest being first. */ + order?: number; + /** @description A width for the column, defined in pixels - this affects rendering in tables. */ + width?: number; + /** @description If this is a relationship column, we can set the columns we wish to include */ + column?: { + readonly?: boolean; + }[]; + } + | { + /** + * @description This column should be built from a calculation, specifying a type and field. It is important to note when a calculation is configured all non-calculation columns will be used for grouping. + * @enum {string} + */ + calculationType?: "sum" | "avg" | "count" | "min" | "max"; + /** @description The field from the table to perform the calculation on. */ + field?: string; + /** @description Can be used in tandem with the count calculation type, to count unique entries. */ + distinct?: boolean; + }; }; /** @description The ID of the view. */ id: string; @@ -1295,6 +1361,31 @@ export interface operations { }; }; }; + rowViewSearch: { + parameters: { + path: { + /** The ID of the view which this request is targeting. */ + viewId: components["parameters"]["viewId"]; + }; + header: { + /** The ID of the app which this request is targeting. */ + "x-budibase-app-id": components["parameters"]["appId"]; + }; + }; + responses: { + /** The response will contain an array of rows that match the search parameters. */ + 200: { + content: { + "application/json": components["schemas"]["searchOutput"]; + }; + }; + }; + requestBody: { + content: { + "application/json": components["schemas"]["rowSearch"]; + }; + }; + }; /** Create a table, this could be internal or external. */ tableCreate: { parameters: { From b3ee2660230f426489b9a6b49860e3af7a1f61a0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Nov 2024 17:53:26 +0000 Subject: [PATCH 025/193] Adding test case for creation, updating validator. --- .../src/api/controllers/public/views.ts | 3 + .../src/api/routes/public/tests/Request.ts | 82 ++++++++++++++++++- .../src/api/routes/public/tests/views.spec.ts | 32 ++++++++ .../server/src/api/routes/utils/validators.ts | 36 +++++++- 4 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 packages/server/src/api/routes/public/tests/views.spec.ts diff --git a/packages/server/src/api/controllers/public/views.ts b/packages/server/src/api/controllers/public/views.ts index e400e850ea..135d3c0128 100644 --- a/packages/server/src/api/controllers/public/views.ts +++ b/packages/server/src/api/controllers/public/views.ts @@ -13,6 +13,9 @@ function fixView(view: ViewV2, params?: { viewId: string }) { if (!view.version) { view.version = 2 } + if (!view.query) { + view.query = {} + } return view } diff --git a/packages/server/src/api/routes/public/tests/Request.ts b/packages/server/src/api/routes/public/tests/Request.ts index 92a4996124..7b3c5b6e06 100644 --- a/packages/server/src/api/routes/public/tests/Request.ts +++ b/packages/server/src/api/routes/public/tests/Request.ts @@ -1,10 +1,19 @@ -import { User, Table, SearchFilters, Row } from "@budibase/types" +import { + User, + Table, + SearchFilters, + Row, + ViewV2Schema, + ViewV2, + ViewV2Type, +} from "@budibase/types" import { HttpMethod, MakeRequestResponse, generateMakeRequest } from "./utils" import TestConfiguration from "../../../../tests/utilities/TestConfiguration" -import { Expectations } from "../../../../tests/utilities/api/base" type RequestOpts = { internal?: boolean; appId?: string } +type Response = { data: T } + export interface PublicAPIExpectations { status?: number body?: Record @@ -15,6 +24,7 @@ export class PublicAPIRequest { private appId: string | undefined tables: PublicTableAPI + views: PublicViewAPI rows: PublicRowAPI apiKey: string @@ -28,6 +38,7 @@ export class PublicAPIRequest { this.appId = appId this.tables = new PublicTableAPI(this) this.rows = new PublicRowAPI(this) + this.views = new PublicViewAPI(this) } static async init(config: TestConfiguration, user: User, opts?: RequestOpts) { @@ -73,7 +84,7 @@ export class PublicTableAPI { async create( table: Table, expectations?: PublicAPIExpectations - ): Promise<{ data: Table }> { + ): Promise> { return this.request.send("post", "/tables", table, expectations) } } @@ -89,7 +100,7 @@ export class PublicRowAPI { tableId: string, query: SearchFilters, expectations?: PublicAPIExpectations - ): Promise<{ data: Row[] }> { + ): Promise> { return this.request.send( "post", `/tables/${tableId}/rows/search`, @@ -99,4 +110,67 @@ export class PublicRowAPI { expectations ) } + + async viewSearch( + viewId: string, + query: SearchFilters, + expectations?: PublicAPIExpectations + ): Promise> { + return this.request.send( + "post", + `/views/${viewId}/rows/search`, + { + query, + }, + expectations + ) + } +} + +export class PublicViewAPI { + request: PublicAPIRequest + + constructor(request: PublicAPIRequest) { + this.request = request + } + + async create( + view: Omit, + expectations?: PublicAPIExpectations + ): Promise> { + return this.request.send("post", "/views", view, expectations) + } + + async update( + viewId: string, + view: Omit, + expectations?: PublicAPIExpectations + ): Promise> { + return this.request.send("put", `/views/${viewId}`, view, expectations) + } + + async destroy( + viewId: string, + expectations?: PublicAPIExpectations + ): Promise { + return this.request.send( + "delete", + `/views/${viewId}`, + undefined, + expectations + ) + } + + async find( + viewId: string, + expectations?: PublicAPIExpectations + ): Promise> { + return this.request.send("get", `/views/${viewId}`, undefined, expectations) + } + + async fetch( + expectations?: PublicAPIExpectations + ): Promise> { + return this.request.send("get", "/views", undefined, expectations) + } } diff --git a/packages/server/src/api/routes/public/tests/views.spec.ts b/packages/server/src/api/routes/public/tests/views.spec.ts new file mode 100644 index 0000000000..dce2e1ab7a --- /dev/null +++ b/packages/server/src/api/routes/public/tests/views.spec.ts @@ -0,0 +1,32 @@ +import * as setup from "../../tests/utilities" +import { basicTable } from "../../../../tests/utilities/structures" +import { Table } from "@budibase/types" +import { PublicAPIRequest } from "./Request" + +describe("check public API security", () => { + const config = setup.getConfig() + let request: PublicAPIRequest, table: Table + + beforeAll(async () => { + await config.init() + request = await PublicAPIRequest.init(config, await config.globalUser()) + table = (await request.tables.create(basicTable())).data + }) + + it("should be able to create a view", async () => { + await request.views.create( + { + name: "view", + tableId: table._id!, + query: {}, + schema: { + name: { + readonly: true, + visible: true, + }, + }, + }, + { status: 201 } + ) + }) +}) diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 68ebd72c5e..9bae8b64d9 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -9,6 +9,9 @@ import { Table, WebhookActionType, BuiltinPermissionID, + ViewV2Type, + SortOrder, + SortType, } from "@budibase/types" import Joi, { CustomValidator } from "joi" import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core" @@ -67,7 +70,26 @@ export function tableValidator() { } export function viewValidator() { - return auth.joiValidator.body(Joi.object()) + return auth.joiValidator.body( + Joi.object({ + id: OPTIONAL_STRING, + tableId: Joi.string().required(), + name: Joi.string().required(), + type: Joi.string().optional().valid(null, ViewV2Type.CALCULATION), + primaryDisplay: OPTIONAL_STRING, + schema: Joi.object().required(), + query: searchFiltersValidator().optional(), + sort: Joi.object({ + field: Joi.string().required(), + order: Joi.string() + .optional() + .valid(...Object.values(SortOrder)), + type: Joi.string() + .optional() + .valid(...Object.values(SortType)), + }).optional(), + }) + ) } export function nameValidator() { @@ -95,8 +117,7 @@ export function datasourceValidator() { ) } -function filterObject(opts?: { unknown: boolean }) { - const { unknown = true } = opts || {} +function searchFiltersValidator() { const conditionalFilteringObject = () => Joi.object({ conditions: Joi.array().items(Joi.link("#schema")).required(), @@ -123,7 +144,14 @@ function filterObject(opts?: { unknown: boolean }) { fuzzyOr: Joi.forbidden(), documentType: Joi.forbidden(), } - return Joi.object(filtersValidators).unknown(unknown).id("schema") + + return Joi.object(filtersValidators) +} + +function filterObject(opts?: { unknown: boolean }) { + const { unknown = true } = opts || {} + + return searchFiltersValidator().unknown(unknown).id("schema") } export function internalSearchValidator() { From 6e5aab1f3f9269f5da24084ed204d5bcb6df3ae5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Nov 2024 17:53:38 +0000 Subject: [PATCH 026/193] Updating view definition to include complete query and sorting. --- packages/server/specs/openapi.json | 348 +++++++++++++++++++++ packages/server/specs/openapi.yaml | 301 ++++++++++++++++++ packages/server/specs/resources/misc.ts | 182 +++++------ packages/server/specs/resources/view.ts | 25 +- packages/server/src/definitions/openapi.ts | 174 +++++++++++ 5 files changed, 939 insertions(+), 91 deletions(-) diff --git a/packages/server/specs/openapi.json b/packages/server/specs/openapi.json index d45a8f28c9..fcabba8eba 100644 --- a/packages/server/specs/openapi.json +++ b/packages/server/specs/openapi.json @@ -2110,6 +2110,122 @@ "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." }, + "query": { + "type": "object", + "properties": { + "allOr": { + "type": "boolean", + "description": "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used." + }, + "string": { + "type": "object", + "example": { + "columnName1": "value", + "columnName2": "value" + }, + "description": "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.", + "additionalProperties": { + "type": "string", + "description": "The value to search for in the column." + } + }, + "fuzzy": { + "type": "object", + "description": "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'." + }, + "range": { + "type": "object", + "description": "Searches within a range, the format of this must be in the format of an object with a \"low\" and \"high\" property.", + "example": { + "columnName1": { + "low": 10, + "high": 20 + } + } + }, + "equal": { + "type": "object", + "description": "Searches for rows that have a column value that is exactly the value set." + }, + "notEqual": { + "type": "object", + "description": "Searches for any row which does not contain the specified column value." + }, + "empty": { + "type": "object", + "description": "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.", + "example": { + "columnName1": "" + } + }, + "notEmpty": { + "type": "object", + "description": "Searches for rows which have the specified column." + }, + "oneOf": { + "type": "object", + "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]." + }, + "contains": { + "type": "object", + "description": "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "notContains": { + "type": "object", + "description": "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "containsAny": { + "type": "object", + "description": "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + } + } + }, + "sort": { + "type": "object", + "required": [ + "field" + ], + "properties": { + "field": { + "type": "string", + "description": "The field from the table/view schema to sort on." + }, + "order": { + "type": "string", + "description": "The order in which to sort.", + "enum": [ + "ascending", + "descending" + ] + }, + "type": { + "type": "string", + "description": "The type of sort to perform (by number, or by alphabetically).", + "enum": [ + "string", + "number" + ] + } + } + }, "schema": { "type": "object", "additionalProperties": { @@ -2203,6 +2319,122 @@ "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." }, + "query": { + "type": "object", + "properties": { + "allOr": { + "type": "boolean", + "description": "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used." + }, + "string": { + "type": "object", + "example": { + "columnName1": "value", + "columnName2": "value" + }, + "description": "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.", + "additionalProperties": { + "type": "string", + "description": "The value to search for in the column." + } + }, + "fuzzy": { + "type": "object", + "description": "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'." + }, + "range": { + "type": "object", + "description": "Searches within a range, the format of this must be in the format of an object with a \"low\" and \"high\" property.", + "example": { + "columnName1": { + "low": 10, + "high": 20 + } + } + }, + "equal": { + "type": "object", + "description": "Searches for rows that have a column value that is exactly the value set." + }, + "notEqual": { + "type": "object", + "description": "Searches for any row which does not contain the specified column value." + }, + "empty": { + "type": "object", + "description": "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.", + "example": { + "columnName1": "" + } + }, + "notEmpty": { + "type": "object", + "description": "Searches for rows which have the specified column." + }, + "oneOf": { + "type": "object", + "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]." + }, + "contains": { + "type": "object", + "description": "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "notContains": { + "type": "object", + "description": "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "containsAny": { + "type": "object", + "description": "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + } + } + }, + "sort": { + "type": "object", + "required": [ + "field" + ], + "properties": { + "field": { + "type": "string", + "description": "The field from the table/view schema to sort on." + }, + "order": { + "type": "string", + "description": "The order in which to sort.", + "enum": [ + "ascending", + "descending" + ] + }, + "type": { + "type": "string", + "description": "The type of sort to perform (by number, or by alphabetically).", + "enum": [ + "string", + "number" + ] + } + } + }, "schema": { "type": "object", "additionalProperties": { @@ -2307,6 +2539,122 @@ "type": "string", "description": "A column used to display rows from this view - usually used when rendered in tables." }, + "query": { + "type": "object", + "properties": { + "allOr": { + "type": "boolean", + "description": "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used." + }, + "string": { + "type": "object", + "example": { + "columnName1": "value", + "columnName2": "value" + }, + "description": "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.", + "additionalProperties": { + "type": "string", + "description": "The value to search for in the column." + } + }, + "fuzzy": { + "type": "object", + "description": "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'." + }, + "range": { + "type": "object", + "description": "Searches within a range, the format of this must be in the format of an object with a \"low\" and \"high\" property.", + "example": { + "columnName1": { + "low": 10, + "high": 20 + } + } + }, + "equal": { + "type": "object", + "description": "Searches for rows that have a column value that is exactly the value set." + }, + "notEqual": { + "type": "object", + "description": "Searches for any row which does not contain the specified column value." + }, + "empty": { + "type": "object", + "description": "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.", + "example": { + "columnName1": "" + } + }, + "notEmpty": { + "type": "object", + "description": "Searches for rows which have the specified column." + }, + "oneOf": { + "type": "object", + "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]." + }, + "contains": { + "type": "object", + "description": "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "notContains": { + "type": "object", + "description": "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + }, + "containsAny": { + "type": "object", + "description": "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", + "example": { + "arrayColumn": [ + "a", + "b" + ] + } + } + } + }, + "sort": { + "type": "object", + "required": [ + "field" + ], + "properties": { + "field": { + "type": "string", + "description": "The field from the table/view schema to sort on." + }, + "order": { + "type": "string", + "description": "The order in which to sort.", + "enum": [ + "ascending", + "descending" + ] + }, + "type": { + "type": "string", + "description": "The type of sort to perform (by number, or by alphabetically).", + "enum": [ + "string", + "number" + ] + } + } + }, "schema": { "type": "object", "additionalProperties": { diff --git a/packages/server/specs/openapi.yaml b/packages/server/specs/openapi.yaml index 0e551207ed..4d2a4fc5bd 100644 --- a/packages/server/specs/openapi.yaml +++ b/packages/server/specs/openapi.yaml @@ -1767,6 +1767,106 @@ components: type: string description: A column used to display rows from this view - usually used when rendered in tables. + query: + type: object + properties: + allOr: + type: boolean + description: Specifies that a row should be returned if it satisfies any of the + specified options, rather than requiring it to fulfill all the + search parameters. This defaults to false, meaning AND logic + will be used. + string: + type: object + example: + columnName1: value + columnName2: value + description: A map of field name to the string to search for, this will look for + rows that have a value starting with the string value. + additionalProperties: + type: string + description: The value to search for in the column. + fuzzy: + type: object + description: Searches for a sub-string within a string column, e.g. searching + for 'dib' will match 'Budibase'. + range: + type: object + description: Searches within a range, the format of this must be in the format + of an object with a "low" and "high" property. + example: + columnName1: + low: 10 + high: 20 + equal: + type: object + description: Searches for rows that have a column value that is exactly the + value set. + notEqual: + type: object + description: Searches for any row which does not contain the specified column + value. + empty: + type: object + description: Searches for rows which do not contain the specified column. The + object should simply contain keys of the column names, these can + map to any value. + example: + columnName1: "" + notEmpty: + type: object + description: Searches for rows which have the specified column. + oneOf: + type: object + description: Searches for rows which have a column value that is any of the + specified values. The format of this must be columnName -> + [value1, value2]. + contains: + type: object + description: Searches for a value, or set of values in array column types (such + as a multi-select). If an array of search options is provided + then it must match all. + example: + arrayColumn: + - a + - b + notContains: + type: object + description: The logical inverse of contains. Only works on array column types. + If an array of values is passed, the row must not match any of + them to be returned in the response. + example: + arrayColumn: + - a + - b + containsAny: + type: object + description: As with the contains search, only works on array column types and + searches for any of the provided values when given an array. + example: + arrayColumn: + - a + - b + sort: + type: object + required: + - field + properties: + field: + type: string + description: The field from the table/view schema to sort on. + order: + type: string + description: The order in which to sort. + enum: + - ascending + - descending + type: + type: string + description: The type of sort to perform (by number, or by alphabetically). + enum: + - string + - number schema: type: object additionalProperties: @@ -1843,6 +1943,106 @@ components: type: string description: A column used to display rows from this view - usually used when rendered in tables. + query: + type: object + properties: + allOr: + type: boolean + description: Specifies that a row should be returned if it satisfies any of the + specified options, rather than requiring it to fulfill all + the search parameters. This defaults to false, meaning AND + logic will be used. + string: + type: object + example: + columnName1: value + columnName2: value + description: A map of field name to the string to search for, this will look for + rows that have a value starting with the string value. + additionalProperties: + type: string + description: The value to search for in the column. + fuzzy: + type: object + description: Searches for a sub-string within a string column, e.g. searching + for 'dib' will match 'Budibase'. + range: + type: object + description: Searches within a range, the format of this must be in the format + of an object with a "low" and "high" property. + example: + columnName1: + low: 10 + high: 20 + equal: + type: object + description: Searches for rows that have a column value that is exactly the + value set. + notEqual: + type: object + description: Searches for any row which does not contain the specified column + value. + empty: + type: object + description: Searches for rows which do not contain the specified column. The + object should simply contain keys of the column names, these + can map to any value. + example: + columnName1: "" + notEmpty: + type: object + description: Searches for rows which have the specified column. + oneOf: + type: object + description: Searches for rows which have a column value that is any of the + specified values. The format of this must be columnName -> + [value1, value2]. + contains: + type: object + description: Searches for a value, or set of values in array column types (such + as a multi-select). If an array of search options is + provided then it must match all. + example: + arrayColumn: + - a + - b + notContains: + type: object + description: The logical inverse of contains. Only works on array column types. + If an array of values is passed, the row must not match any + of them to be returned in the response. + example: + arrayColumn: + - a + - b + containsAny: + type: object + description: As with the contains search, only works on array column types and + searches for any of the provided values when given an array. + example: + arrayColumn: + - a + - b + sort: + type: object + required: + - field + properties: + field: + type: string + description: The field from the table/view schema to sort on. + order: + type: string + description: The order in which to sort. + enum: + - ascending + - descending + type: + type: string + description: The type of sort to perform (by number, or by alphabetically). + enum: + - string + - number schema: type: object additionalProperties: @@ -1926,6 +2126,107 @@ components: type: string description: A column used to display rows from this view - usually used when rendered in tables. + query: + type: object + properties: + allOr: + type: boolean + description: Specifies that a row should be returned if it satisfies any of the + specified options, rather than requiring it to fulfill all + the search parameters. This defaults to false, meaning AND + logic will be used. + string: + type: object + example: + columnName1: value + columnName2: value + description: A map of field name to the string to search for, this will look for + rows that have a value starting with the string value. + additionalProperties: + type: string + description: The value to search for in the column. + fuzzy: + type: object + description: Searches for a sub-string within a string column, e.g. searching + for 'dib' will match 'Budibase'. + range: + type: object + description: Searches within a range, the format of this must be in the format + of an object with a "low" and "high" property. + example: + columnName1: + low: 10 + high: 20 + equal: + type: object + description: Searches for rows that have a column value that is exactly the + value set. + notEqual: + type: object + description: Searches for any row which does not contain the specified column + value. + empty: + type: object + description: Searches for rows which do not contain the specified column. The + object should simply contain keys of the column names, + these can map to any value. + example: + columnName1: "" + notEmpty: + type: object + description: Searches for rows which have the specified column. + oneOf: + type: object + description: Searches for rows which have a column value that is any of the + specified values. The format of this must be columnName -> + [value1, value2]. + contains: + type: object + description: Searches for a value, or set of values in array column types (such + as a multi-select). If an array of search options is + provided then it must match all. + example: + arrayColumn: + - a + - b + notContains: + type: object + description: The logical inverse of contains. Only works on array column types. + If an array of values is passed, the row must not match + any of them to be returned in the response. + example: + arrayColumn: + - a + - b + containsAny: + type: object + description: As with the contains search, only works on array column types and + searches for any of the provided values when given an + array. + example: + arrayColumn: + - a + - b + sort: + type: object + required: + - field + properties: + field: + type: string + description: The field from the table/view schema to sort on. + order: + type: string + description: The order in which to sort. + enum: + - ascending + - descending + type: + type: string + description: The type of sort to perform (by number, or by alphabetically). + enum: + - string + - number schema: type: object additionalProperties: diff --git a/packages/server/specs/resources/misc.ts b/packages/server/specs/resources/misc.ts index f56dff3301..8f77d2b22a 100644 --- a/packages/server/specs/resources/misc.ts +++ b/packages/server/specs/resources/misc.ts @@ -1,99 +1,101 @@ import { object } from "./utils" import Resource from "./utils/Resource" +export const searchSchema = { + type: "object", + properties: { + allOr: { + type: "boolean", + description: + "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.", + }, + string: { + type: "object", + example: { + columnName1: "value", + columnName2: "value", + }, + description: + "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.", + additionalProperties: { + type: "string", + description: "The value to search for in the column.", + }, + }, + fuzzy: { + type: "object", + description: + "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.", + }, + range: { + type: "object", + description: + 'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.', + example: { + columnName1: { + low: 10, + high: 20, + }, + }, + }, + equal: { + type: "object", + description: + "Searches for rows that have a column value that is exactly the value set.", + }, + notEqual: { + type: "object", + description: + "Searches for any row which does not contain the specified column value.", + }, + empty: { + type: "object", + description: + "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.", + example: { + columnName1: "", + }, + }, + notEmpty: { + type: "object", + description: "Searches for rows which have the specified column.", + }, + oneOf: { + type: "object", + description: + "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].", + }, + contains: { + type: "object", + description: + "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", + example: { + arrayColumn: ["a", "b"], + }, + }, + notContains: { + type: "object", + description: + "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", + example: { + arrayColumn: ["a", "b"], + }, + }, + containsAny: { + type: "object", + description: + "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", + example: { + arrayColumn: ["a", "b"], + }, + }, + }, +} + export default new Resource().setSchemas({ rowSearch: object( { - query: { - type: "object", - properties: { - allOr: { - type: "boolean", - description: - "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.", - }, - string: { - type: "object", - example: { - columnName1: "value", - columnName2: "value", - }, - description: - "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.", - additionalProperties: { - type: "string", - description: "The value to search for in the column.", - }, - }, - fuzzy: { - type: "object", - description: - "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.", - }, - range: { - type: "object", - description: - 'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.', - example: { - columnName1: { - low: 10, - high: 20, - }, - }, - }, - equal: { - type: "object", - description: - "Searches for rows that have a column value that is exactly the value set.", - }, - notEqual: { - type: "object", - description: - "Searches for any row which does not contain the specified column value.", - }, - empty: { - type: "object", - description: - "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.", - example: { - columnName1: "", - }, - }, - notEmpty: { - type: "object", - description: "Searches for rows which have the specified column.", - }, - oneOf: { - type: "object", - description: - "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].", - }, - contains: { - type: "object", - description: - "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.", - example: { - arrayColumn: ["a", "b"], - }, - }, - notContains: { - type: "object", - description: - "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.", - example: { - arrayColumn: ["a", "b"], - }, - }, - containsAny: { - type: "object", - description: - "As with the contains search, only works on array column types and searches for any of the provided values when given an array.", - example: { - arrayColumn: ["a", "b"], - }, - }, - }, - }, + query: searchSchema, paginate: { type: "boolean", description: "Enables pagination, by default this is disabled.", diff --git a/packages/server/specs/resources/view.ts b/packages/server/specs/resources/view.ts index e798883ced..364e7badd2 100644 --- a/packages/server/specs/resources/view.ts +++ b/packages/server/specs/resources/view.ts @@ -1,6 +1,7 @@ import { object } from "./utils" import Resource from "./utils/Resource" -import { CalculationType } from "@budibase/types" +import { CalculationType, SortOrder, SortType } from "@budibase/types" +import { searchSchema } from "./misc" const view = { name: "peopleView", @@ -81,6 +82,28 @@ const viewSchema = { description: "A column used to display rows from this view - usually used when rendered in tables.", }, + query: searchSchema, + sort: { + type: "object", + required: ["field"], + properties: { + field: { + type: "string", + description: "The field from the table/view schema to sort on.", + }, + order: { + type: "string", + description: "The order in which to sort.", + enum: Object.values(SortOrder), + }, + type: { + type: "string", + description: + "The type of sort to perform (by number, or by alphabetically).", + enum: Object.values(SortType), + }, + }, + }, schema: { type: "object", additionalProperties: { diff --git a/packages/server/src/definitions/openapi.ts b/packages/server/src/definitions/openapi.ts index fed19ec5cf..6f53df0ab5 100644 --- a/packages/server/src/definitions/openapi.ts +++ b/packages/server/src/definitions/openapi.ts @@ -843,6 +843,64 @@ export interface components { type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; + query?: { + /** @description Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used. */ + allOr?: boolean; + /** + * @description A map of field name to the string to search for, this will look for rows that have a value starting with the string value. + * @example [object Object] + */ + string?: { [key: string]: string }; + /** @description Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'. */ + fuzzy?: { [key: string]: unknown }; + /** + * @description Searches within a range, the format of this must be in the format of an object with a "low" and "high" property. + * @example [object Object] + */ + range?: { [key: string]: unknown }; + /** @description Searches for rows that have a column value that is exactly the value set. */ + equal?: { [key: string]: unknown }; + /** @description Searches for any row which does not contain the specified column value. */ + notEqual?: { [key: string]: unknown }; + /** + * @description Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value. + * @example [object Object] + */ + empty?: { [key: string]: unknown }; + /** @description Searches for rows which have the specified column. */ + notEmpty?: { [key: string]: unknown }; + /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */ + oneOf?: { [key: string]: unknown }; + /** + * @description Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all. + * @example [object Object] + */ + contains?: { [key: string]: unknown }; + /** + * @description The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response. + * @example [object Object] + */ + notContains?: { [key: string]: unknown }; + /** + * @description As with the contains search, only works on array column types and searches for any of the provided values when given an array. + * @example [object Object] + */ + containsAny?: { [key: string]: unknown }; + }; + sort?: { + /** @description The field from the table/view schema to sort on. */ + field: string; + /** + * @description The order in which to sort. + * @enum {string} + */ + order?: "ascending" | "descending"; + /** + * @description The type of sort to perform (by number, or by alphabetically). + * @enum {string} + */ + type?: "string" | "number"; + }; schema: { [key: string]: | { @@ -884,6 +942,64 @@ export interface components { type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; + query?: { + /** @description Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used. */ + allOr?: boolean; + /** + * @description A map of field name to the string to search for, this will look for rows that have a value starting with the string value. + * @example [object Object] + */ + string?: { [key: string]: string }; + /** @description Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'. */ + fuzzy?: { [key: string]: unknown }; + /** + * @description Searches within a range, the format of this must be in the format of an object with a "low" and "high" property. + * @example [object Object] + */ + range?: { [key: string]: unknown }; + /** @description Searches for rows that have a column value that is exactly the value set. */ + equal?: { [key: string]: unknown }; + /** @description Searches for any row which does not contain the specified column value. */ + notEqual?: { [key: string]: unknown }; + /** + * @description Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value. + * @example [object Object] + */ + empty?: { [key: string]: unknown }; + /** @description Searches for rows which have the specified column. */ + notEmpty?: { [key: string]: unknown }; + /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */ + oneOf?: { [key: string]: unknown }; + /** + * @description Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all. + * @example [object Object] + */ + contains?: { [key: string]: unknown }; + /** + * @description The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response. + * @example [object Object] + */ + notContains?: { [key: string]: unknown }; + /** + * @description As with the contains search, only works on array column types and searches for any of the provided values when given an array. + * @example [object Object] + */ + containsAny?: { [key: string]: unknown }; + }; + sort?: { + /** @description The field from the table/view schema to sort on. */ + field: string; + /** + * @description The order in which to sort. + * @enum {string} + */ + order?: "ascending" | "descending"; + /** + * @description The type of sort to perform (by number, or by alphabetically). + * @enum {string} + */ + type?: "string" | "number"; + }; schema: { [key: string]: | { @@ -927,6 +1043,64 @@ export interface components { type?: "calculation"; /** @description A column used to display rows from this view - usually used when rendered in tables. */ primaryDisplay?: string; + query?: { + /** @description Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used. */ + allOr?: boolean; + /** + * @description A map of field name to the string to search for, this will look for rows that have a value starting with the string value. + * @example [object Object] + */ + string?: { [key: string]: string }; + /** @description Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'. */ + fuzzy?: { [key: string]: unknown }; + /** + * @description Searches within a range, the format of this must be in the format of an object with a "low" and "high" property. + * @example [object Object] + */ + range?: { [key: string]: unknown }; + /** @description Searches for rows that have a column value that is exactly the value set. */ + equal?: { [key: string]: unknown }; + /** @description Searches for any row which does not contain the specified column value. */ + notEqual?: { [key: string]: unknown }; + /** + * @description Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value. + * @example [object Object] + */ + empty?: { [key: string]: unknown }; + /** @description Searches for rows which have the specified column. */ + notEmpty?: { [key: string]: unknown }; + /** @description Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]. */ + oneOf?: { [key: string]: unknown }; + /** + * @description Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all. + * @example [object Object] + */ + contains?: { [key: string]: unknown }; + /** + * @description The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response. + * @example [object Object] + */ + notContains?: { [key: string]: unknown }; + /** + * @description As with the contains search, only works on array column types and searches for any of the provided values when given an array. + * @example [object Object] + */ + containsAny?: { [key: string]: unknown }; + }; + sort?: { + /** @description The field from the table/view schema to sort on. */ + field: string; + /** + * @description The order in which to sort. + * @enum {string} + */ + order?: "ascending" | "descending"; + /** + * @description The type of sort to perform (by number, or by alphabetically). + * @enum {string} + */ + type?: "string" | "number"; + }; schema: { [key: string]: | { From 7235fd9d5ce012a845e77b5690ca3e475afea529 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 4 Nov 2024 10:25:38 +0100 Subject: [PATCH 027/193] Merge from master --- .github/workflows/budibase_ci.yml | 24 + .github/workflows/deploy-qa.yml | 2 +- .gitignore | 3 +- hosting/single/Dockerfile | 3 +- lerna.json | 2 +- packages/account-portal | 2 +- packages/backend-core/src/constants/misc.ts | 1 + packages/backend-core/src/context/identity.ts | 2 +- packages/backend-core/src/environment.ts | 4 +- .../backend-core/src/events/identification.ts | 13 +- .../src/middleware/authenticated.ts | 50 +- .../backend-core/src/security/permissions.ts | 14 +- packages/backend-core/src/security/roles.ts | 66 +- .../src/security/tests/permissions.spec.ts | 7 +- packages/backend-core/src/sql/sql.ts | 572 ++++---- packages/backend-core/src/tenancy/db.ts | 23 - packages/backend-core/src/users/db.ts | 21 +- packages/backend-core/src/users/users.ts | 11 + packages/backend-core/src/users/utils.ts | 30 +- .../tests/core/utilities/mocks/licenses.ts | 8 + .../bbui/src/ActionButton/ActionButton.svelte | 127 +- .../bbui/src/ActionMenu/ActionMenu.svelte | 41 +- .../bbui/src/Actions/position_dropdown.js | 12 +- packages/bbui/src/Button/Button.svelte | 15 +- .../ButtonGroup/CollapsedButtonGroup.svelte | 57 + packages/bbui/src/Form/Core/Switch.svelte | 1 + packages/bbui/src/Form/Core/TextField.svelte | 9 +- packages/bbui/src/Icon/Icon.svelte | 5 +- .../bbui/src/InlineAlert/InlineAlert.svelte | 1 + packages/bbui/src/List/ListItem.svelte | 158 ++- packages/bbui/src/Menu/Item.svelte | 7 +- packages/bbui/src/Modal/ModalContent.svelte | 7 +- .../bbui/src/Notification/Notification.svelte | 6 +- packages/bbui/src/Popover/Popover.svelte | 35 +- packages/bbui/src/helpers.js | 10 + packages/bbui/src/index.js | 2 + packages/builder/package.json | 2 + .../FlowChart/FlowChart.svelte | 28 +- .../FlowChart/FlowItem.svelte | 16 +- .../AutomationPanel/AutomationNavItem.svelte | 106 +- .../AutomationPanel/AutomationPanel.svelte | 78 +- .../CreateAutomationModal.svelte | 4 +- .../SetupPanel/AutomationBlockSetup.svelte | 38 +- .../automation/SetupPanel/CronBuilder.svelte | 5 +- .../automation/SetupPanel/RowSelector.svelte | 10 +- .../DataTable/RelationshipDataTable.svelte | 44 - .../backend/DataTable/TableDataTable.svelte | 120 -- .../backend/DataTable/ViewDataTable.svelte | 80 -- .../backend/DataTable/ViewV2DataTable.svelte | 58 - .../DataTable/buttons/EditRolesButton.svelte | 13 - .../DataTable/buttons/ExportButton.svelte | 144 +- .../DataTable/buttons/ImportButton.svelte | 82 +- .../buttons/ManageAccessButton.svelte | 207 ++- .../buttons/TableFilterButton.svelte | 84 +- .../grid}/ColumnsSettingContent.svelte | 47 +- .../buttons/grid/GridAutomationsButton.svelte | 75 + .../grid/GridColumnsSettingButton.svelte | 54 + .../grid/GridCreateAutomationButton.svelte | 101 -- .../buttons/grid/GridCreateViewButton.svelte | 29 - .../buttons/grid/GridExportButton.svelte | 28 +- .../buttons/grid/GridGenerateButton.svelte | 191 +++ .../grid/GridManageAccessButton.svelte | 10 +- .../buttons/grid/GridRowActionsButton.svelte | 146 ++ .../buttons/grid/GridScreensButton.svelte | 59 + .../buttons/grid/GridSizeButton.svelte | 127 ++ .../buttons/grid/GridSortButton.svelte | 79 ++ .../grid/GridUsersTableButton.svelte | 0 .../grid/GridViewCalculationButton.svelte | 267 ++++ .../DataTable/buttons/grid/magic-wand.svg | 6 + .../components/backend/DataTable/formula.js | 1 + .../DataTable/modals/CreateEditColumn.svelte | 182 ++- .../DataTable/modals/CreateEditRow.svelte | 3 +- .../DataTable/modals/CreateEditUser.svelte | 2 +- .../backend/DataTable/modals/EditRoles.svelte | 174 --- .../DataTable/modals/ExportModal.svelte | 224 --- .../DataTable/modals/ExportModal.test.js | 241 ---- .../DataTable/modals/ImportModal.svelte | 61 - .../DataTable/modals/ManageAccessModal.svelte | 155 --- .../modals/grid/GridCreateViewModal.svelte | 60 - .../DatasourceNavigator.svelte | 11 +- .../backend/RoleEditor/BracketEdge.svelte | 63 + .../backend/RoleEditor/Controls.svelte | 74 + .../backend/RoleEditor/EmptyStateNode.svelte | 24 + .../backend/RoleEditor/RoleEdge.svelte | 123 ++ .../backend/RoleEditor/RoleEditor.svelte | 8 + .../backend/RoleEditor/RoleFlow.svelte | 234 ++++ .../backend/RoleEditor/RoleNode.svelte | 231 ++++ .../backend/RoleEditor/constants.js | 9 + .../components/backend/RoleEditor/utils.js | 245 ++++ .../ExistingTableDataImport.svelte | 183 ++- .../TableNavigator/TableDataImport.svelte | 139 +- .../DeleteConfirmationModal.svelte | 2 +- .../TableNavItem/TableNavItem.svelte | 16 +- .../TableNavigator/TableNavigator.svelte | 20 +- .../ViewNavItem/ViewNavItem.svelte | 71 - .../modals/CreateTableModal.svelte | 30 +- .../backend/TableNavigator/utils.js | 4 + .../commandPalette/CommandPalette.svelte | 8 +- .../common/AIFieldConfiguration.svelte | 59 + .../components/common/DetailPopover.svelte | 77 ++ .../src/components/common/RoleIcon.svelte | 6 +- .../src/components/common/RoleSelect.svelte | 10 +- .../common}/ToggleActionButtonGroup.svelte | 9 +- .../src/components/deploy/AppActions.svelte | 20 +- .../actions/RowAction.svelte | 104 ++ .../ButtonActionEditor/actions/SaveRow.svelte | 2 +- .../ButtonActionEditor/actions/index.js | 1 + .../controls/ButtonActionEditor/manifest.json | 5 + .../ButtonConfiguration.svelte | 45 +- .../controls/Explanation/lines/Column.svelte | 3 + .../FilterEditor/FilterBuilder.svelte | 83 +- .../controls/FilterEditor/FilterEditor.svelte | 35 +- .../controls/FormStepConfiguration.svelte | 2 + .../GridColumnConfiguration.svelte | 30 +- .../settings/controls/RoleSelect.svelte | 7 +- .../controls/TableConditionEditor.svelte | 5 +- .../integration/AccessLevelSelect.svelte | 14 +- .../components/integration/QueryEditor.svelte | 3 +- .../integration/RestQueryViewer.svelte | 2 +- .../src/components/settings/ThemeModal.svelte | 6 +- .../components/start/CreateAppModal.svelte | 13 +- .../builder/src/constants/backend/index.js | 6 + packages/builder/src/constants/index.js | 6 + packages/builder/src/dataBinding.js | 4 +- packages/builder/src/helpers/components.js | 2 +- .../_components/BuilderSidePanel.svelte | 23 +- .../builder/app/[application]/_layout.svelte | 3 +- .../app/[application]/data/roles.svelte | 8 + .../[tableId]}/[viewId]/_layout.svelte | 2 +- .../table/[tableId]/[viewId]/index.svelte | 78 ++ .../_components/CreateViewButton.svelte | 133 ++ .../_components/DeleteViewModal.svelte} | 1 + .../_components}/EditViewModal.svelte | 4 +- .../[tableId]/_components/ViewNavBar.svelte | 384 ++++++ .../data/table/[tableId]/_layout.svelte | 16 +- .../data/table/[tableId]/index.svelte | 158 ++- .../relationship/[rowId]/[field]/index.svelte | 10 - .../relationship/[rowId]/index.svelte | 7 - .../table/[tableId]/relationship/index.svelte | 7 - .../[tableId]}/v1/[viewName]/_layout.svelte | 0 .../[tableId]/v1/[viewName]/index.svelte | 91 ++ .../{view => table/[tableId]}/v1/index.svelte | 0 .../app/[application]/data/view/index.svelte | 19 - .../data/view/v1/[viewName]/index.svelte | 18 - .../data/view/v2/[viewId]/index.svelte | 5 - .../[application]/data/view/v2/index.svelte | 5 - .../Component/ComponentSettingsSection.svelte | 11 + .../_components/Component/InfoDisplay.svelte | 37 +- .../_components/Screen/AppThemeSelect.svelte | 17 +- .../[screenId]/_components/AppPreview.svelte | 7 +- .../ScreenList/RoleIndicator.svelte | 6 +- .../NewScreen/CreateScreenModal.svelte | 82 +- .../NewScreen/DatasourceModal.svelte | 38 +- .../_components/NewScreen/TypeModal.svelte | 4 +- .../design/_components/NewScreen/utils.js | 17 + .../settings/automations/index.svelte | 4 + .../builder/portal/apps/[appId]/index.svelte | 8 +- .../builder/portal/settings/ai/index.svelte | 6 +- .../_components/GroupAppsTableRenderer.svelte | 7 +- .../portal/users/users/[userId].svelte | 2 +- .../_components/AppRoleTableRenderer.svelte | 13 +- .../_components/RolesTagsTableRenderer.svelte | 9 - .../_components/TagsTableRenderer.svelte | 35 - .../users/_components/UpdateRolesModal.svelte | 74 - .../builder/portal/users/users/index.svelte | 39 +- packages/builder/src/stores/BudiStore.js | 5 +- .../builder/src/stores/builder/automations.js | 34 +- .../builder/src/stores/builder/components.js | 27 +- packages/builder/src/stores/builder/index.js | 7 +- .../builder/src/stores/builder/published.js | 13 + packages/builder/src/stores/builder/roles.js | 56 +- .../builder/src/stores/builder/rowActions.js | 151 +++ packages/builder/src/stores/builder/theme.js | 19 +- packages/builder/src/stores/builder/views.js | 1 + .../builder/src/stores/builder/websocket.js | 11 +- .../builder/src/stores/portal/featureFlags.js | 16 + packages/builder/src/stores/portal/index.js | 1 + .../builder/src/stores/portal/licensing.js | 9 - packages/builder/src/stores/portal/theme.js | 47 +- packages/builder/src/stores/portal/users.js | 11 +- .../builder/src/templates/BaseStructure.js | 4 +- packages/builder/src/templates/rowActions.js | 99 ++ .../src/templates/screenTemplating/form.js | 47 +- .../templates/screenTemplating/table/index.js | 10 +- .../screenTemplating/table/inline.js | 16 +- .../templates/screenTemplating/table/modal.js | 48 +- .../screenTemplating/table/newScreen.js | 64 +- .../screenTemplating/table/sidePanel.js | 37 +- packages/client/manifest.json | 85 +- .../client/src/components/ClientApp.svelte | 3 +- .../client/src/components/Component.svelte | 7 +- .../src/components/app/ButtonGroup.svelte | 54 +- .../src/components/app/DataProvider.svelte | 32 +- .../src/components/app/GridBlock.svelte | 13 +- .../app/blocks/MultiStepFormblock.svelte | 10 +- .../app/blocks/form/FormBlock.svelte | 4 + .../app/blocks/form/InnerFormBlock.svelte | 8 +- .../app/dynamic-filter/DynamicFilter.svelte | 40 +- .../app/dynamic-filter/FilterModal.svelte | 14 - .../src/components/app/forms/Form.svelte | 18 +- .../src/components/app/forms/InnerForm.svelte | 6 +- .../app/forms/RelationshipField.svelte | 3 + .../src/components/app/forms/validation.js | 8 +- .../components/devtools/DevToolsHeader.svelte | 19 +- packages/client/src/sdk.js | 6 +- packages/client/src/stores/features.js | 6 + packages/client/src/stores/theme.js | 6 +- packages/client/src/utils/buttonActions.js | 14 + packages/client/src/utils/schema.js | 39 +- packages/frontend-core/src/api/automations.js | 9 +- packages/frontend-core/src/api/index.js | 2 + packages/frontend-core/src/api/rowActions.js | 90 ++ packages/frontend-core/src/api/tables.js | 2 +- packages/frontend-core/src/api/user.js | 10 +- packages/frontend-core/src/api/viewsV2.js | 1 + .../src/components/CoreFilterBuilder.svelte | 532 ++++++++ .../src/components/FilterBuilder.svelte | 379 ------ .../src/components/FilterField.svelte | 319 +++++ .../src/components/FilterUsers.svelte | 3 +- .../src/components/grid/cells/AICell.svelte | 99 ++ .../src/components/grid/cells/DataCell.svelte | 2 +- .../src/components/grid/cells/GridCell.svelte | 7 +- .../components/grid/cells/HeaderCell.svelte | 31 +- .../components/grid/cells/NumberCell.svelte | 24 +- .../grid/cells/RelationshipCell.svelte | 4 +- .../src/components/grid/cells/RoleCell.svelte | 45 + .../src/components/grid/cells/TextCell.svelte | 4 +- .../grid/controls/ColumnsSettingButton.svelte | 41 - .../grid/controls/SizeButton.svelte | 136 -- .../grid/controls/SortButton.svelte | 96 -- .../grid/layout/ButtonColumn.svelte | 103 +- .../src/components/grid/layout/Grid.svelte | 49 +- .../src/components/grid/lib/constants.js | 2 + .../src/components/grid/lib/renderers.js | 9 + .../src/components/grid/lib/utils.js | 44 +- .../src/components/grid/lib/websocket.js | 9 + .../grid/overlays/GridPopover.svelte | 6 +- .../grid/overlays/MenuOverlay.svelte | 19 + .../src/components/grid/stores/columns.js | 3 +- .../src/components/grid/stores/config.js | 15 +- .../src/components/grid/stores/datasource.js | 37 +- .../grid/stores/datasources/nonPlus.js | 5 +- .../grid/stores/datasources/table.js | 5 +- .../grid/stores/datasources/viewV2.js | 85 +- .../src/components/grid/stores/filter.js | 32 +- .../src/components/grid/stores/rows.js | 17 +- .../src/components/grid/stores/sort.js | 10 +- .../frontend-core/src/components/index.js | 2 +- packages/frontend-core/src/constants.js | 47 +- packages/frontend-core/src/fetch/DataFetch.js | 26 +- .../frontend-core/src/fetch/TableFetch.js | 3 +- packages/frontend-core/src/fetch/UserFetch.js | 17 +- .../frontend-core/src/fetch/ViewV2Fetch.js | 31 +- packages/frontend-core/src/utils/index.js | 2 +- .../frontend-core/src/utils/relatedColumns.js | 16 +- packages/frontend-core/src/utils/roles.js | 13 - packages/frontend-core/src/utils/schema.js | 24 + packages/frontend-core/src/utils/table.js | 12 +- packages/frontend-core/src/utils/theme.js | 12 - packages/frontend-core/src/utils/utils.js | 2 +- packages/pro | 2 +- packages/server/Dockerfile | 3 +- packages/server/nodemon.json | 2 +- packages/server/scripts/load/utils.js | 2 +- packages/server/specs/generate.ts | 18 +- packages/server/specs/openapi.json | 32 +- packages/server/specs/openapi.yaml | 24 +- packages/server/specs/parameters.ts | 2 + packages/server/specs/resources/index.ts | 1 + packages/server/specs/resources/table.ts | 2 +- .../server/src/api/controllers/application.ts | 135 +- .../server/src/api/controllers/automation.ts | 11 +- .../server/src/api/controllers/datasource.ts | 2 + packages/server/src/api/controllers/role.ts | 55 +- .../api/controllers/row/ExternalRequest.ts | 108 +- .../server/src/api/controllers/row/index.ts | 36 +- .../src/api/controllers/row/staticFormula.ts | 19 +- .../src/api/controllers/row/utils/sqlUtils.ts | 1 + .../server/src/api/controllers/row/views.ts | 2 +- .../src/api/controllers/rowAction/crud.ts | 57 +- .../src/api/controllers/rowAction/run.ts | 2 +- .../src/api/controllers/static/index.ts | 25 +- .../src/api/controllers/table/bulkFormula.ts | 2 +- .../server/src/api/controllers/table/index.ts | 8 +- .../api/controllers/table/tests/utils.spec.ts | 62 +- .../server/src/api/controllers/table/utils.ts | 18 +- .../src/api/controllers/view/exporters.ts | 30 +- .../src/api/controllers/view/viewsV2.ts | 30 +- .../server/src/api/routes/public/index.ts | 2 + .../src/api/routes/public/tests/Request.ts | 102 ++ .../{metrics.spec.js => metrics.spec.ts} | 2 +- .../api/routes/public/tests/security.spec.ts | 71 + .../src/api/routes/public/tests/utils.ts | 27 +- packages/server/src/api/routes/rowAction.ts | 6 - .../src/api/routes/tests/application.spec.ts | 37 +- .../src/api/routes/tests/automation.spec.ts | 66 +- .../src/api/routes/tests/permissions.spec.ts | 15 +- .../routes/tests/queries/generic-sql.spec.ts | 101 +- .../server/src/api/routes/tests/role.spec.ts | 102 +- .../{routing.spec.js => routing.spec.ts} | 61 +- .../server/src/api/routes/tests/row.spec.ts | 497 ++++++- .../src/api/routes/tests/rowAction.spec.ts | 208 ++- .../src/api/routes/tests/screen.spec.ts | 28 +- .../src/api/routes/tests/search.spec.ts | 248 +++- .../src/api/routes/tests/static.spec.ts | 18 + .../src/api/routes/tests/viewV2.spec.ts | 1205 +++++++++++++++-- .../server/src/api/routes/utils/validators.ts | 13 +- packages/server/src/automations/actions.ts | 14 +- .../tests/scenarios/scenarios.spec.ts | 34 + .../tests/utilities/AutomationTestBuilder.ts | 7 +- packages/server/src/automations/triggers.ts | 8 +- packages/server/src/constants/index.ts | 2 - packages/server/src/constants/themes.ts | 19 +- packages/server/src/db/linkedRows/index.ts | 15 +- .../server/src/definitions/automations.ts | 8 +- packages/server/src/definitions/openapi.ts | 15 +- .../server/src/events/AutomationEmitter.ts | 30 +- packages/server/src/events/BudibaseEmitter.ts | 34 +- packages/server/src/events/utils.ts | 6 +- .../src/integration-test/postgres.spec.ts | 52 +- packages/server/src/integrations/mysql.ts | 12 +- .../server/src/integrations/tests/sql.spec.ts | 6 +- packages/server/src/middleware/currentapp.ts | 23 +- ...{currentapp.spec.js => currentapp.spec.ts} | 16 +- packages/server/src/middleware/utils.ts | 13 +- .../src/sdk/app/applications/applications.ts | 8 +- .../server/src/sdk/app/automations/crud.ts | 14 +- .../sdk/app/automations/tests/index.spec.ts | 21 +- .../server/src/sdk/app/automations/utils.ts | 66 +- .../server/src/sdk/app/datasources/plus.ts | 10 + .../app/{rowActions.ts => rowActions/crud.ts} | 85 +- .../server/src/sdk/app/rowActions/index.ts | 2 + .../server/src/sdk/app/rowActions/utils.ts | 9 + .../server/src/sdk/app/rows/queryUtils.ts | 6 +- packages/server/src/sdk/app/rows/search.ts | 15 +- .../src/sdk/app/rows/search/internal/sqs.ts | 1 + packages/server/src/sdk/app/tables/getters.ts | 10 + packages/server/src/sdk/app/views/external.ts | 18 +- packages/server/src/sdk/app/views/index.ts | 87 +- packages/server/src/sdk/app/views/internal.ts | 18 +- packages/server/src/sdk/app/views/utils.ts | 43 + packages/server/src/sdk/users/utils.ts | 3 +- .../src/tests/utilities/TestConfiguration.ts | 5 + .../src/tests/utilities/api/application.ts | 4 +- .../src/tests/utilities/api/automation.ts | 11 - .../src/tests/utilities/api/rowAction.ts | 26 +- .../server/src/tests/utilities/structures.ts | 36 +- packages/server/src/threads/automation.ts | 19 +- packages/server/src/utilities/csv.ts | 6 +- .../server/src/utilities/fileSystem/app.ts | 6 +- .../src/utilities/fileSystem/clientLibrary.ts | 17 + packages/server/src/utilities/index.ts | 36 +- .../src/utilities/rowProcessor/index.ts | 33 +- .../rowProcessor/tests/utils.spec.ts | 75 +- .../src/utilities/rowProcessor/utils.ts | 64 + .../server/src/utilities/tests/utils.spec.ts | 34 + packages/server/src/websockets/builder.ts | 15 + packages/server/src/websockets/grid.ts | 6 +- packages/shared-core/src/constants/ai.ts | 68 + packages/shared-core/src/constants/index.ts | 3 + packages/shared-core/src/constants/themes.ts | 28 + packages/shared-core/src/filters.ts | 429 +++--- .../src/helpers/tests/roles.spec.ts | 4 +- packages/shared-core/src/index.ts | 1 + packages/shared-core/src/table.ts | 4 +- packages/shared-core/src/tests/themes.ts | 28 + packages/shared-core/src/themes.ts | 44 + packages/shared-core/src/utils.ts | 136 +- packages/types/package.json | 3 +- packages/types/src/api/web/app/rowAction.ts | 12 +- packages/types/src/api/web/application.ts | 4 +- packages/types/src/api/web/automation.ts | 16 - packages/types/src/api/web/role.ts | 4 +- packages/types/src/api/web/searchFilter.ts | 60 +- packages/types/src/api/web/user.ts | 5 +- packages/types/src/core/events.ts | 8 + packages/types/src/core/index.ts | 1 + .../documents/app/automation/automation.ts | 1 + .../src/documents/app/automation/schema.ts | 8 +- packages/types/src/documents/app/index.ts | 1 + packages/types/src/documents/app/role.ts | 4 +- packages/types/src/documents/app/rowAction.ts | 1 - .../types/src/documents/app/table/schema.ts | 14 + packages/types/src/documents/app/theme.ts | 14 + packages/types/src/documents/app/view.ts | 6 +- packages/types/src/documents/global/index.ts | 1 - .../types/src/documents/global/tenantInfo.ts | 15 - packages/types/src/documents/global/user.ts | 15 + packages/types/src/sdk/automations/index.ts | 8 +- packages/types/src/sdk/koa.ts | 50 +- packages/types/src/sdk/permissions.ts | 11 + packages/types/src/sdk/row.ts | 2 +- packages/types/src/sdk/search.ts | 18 +- packages/worker/Dockerfile | 3 +- .../src/api/controllers/global/configs.ts | 56 +- .../src/api/controllers/global/tenant.ts | 14 - .../controllers/global/tests/configs.spec.ts | 4 - .../src/api/controllers/global/users.ts | 39 +- packages/worker/src/api/index.ts | 10 +- .../worker/src/api/routes/global/tenant.ts | 11 - .../src/api/routes/global/tests/roles.spec.ts | 6 +- .../api/routes/global/tests/tenant.spec.ts | 48 - .../src/api/routes/global/tests/users.spec.ts | 76 +- .../worker/src/api/routes/global/users.ts | 1 + packages/worker/src/api/routes/index.ts | 2 - .../worker/src/api/routes/validation/users.ts | 9 +- packages/worker/src/tests/api/tenants.ts | 9 - packages/worker/src/tests/api/users.ts | 10 +- yarn.lock | 435 +++++- 409 files changed, 12538 insertions(+), 5700 deletions(-) create mode 100644 packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/RelationshipDataTable.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/TableDataTable.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/ViewDataTable.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte rename packages/{frontend-core/src/components/grid/controls => builder/src/components/backend/DataTable/buttons/grid}/ColumnsSettingContent.svelte (87%) create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridAutomationsButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateViewButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridGenerateButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridRowActionsButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte rename packages/builder/src/components/backend/DataTable/{modals => buttons}/grid/GridUsersTableButton.svelte (100%) create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte create mode 100644 packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg delete mode 100644 packages/builder/src/components/backend/DataTable/modals/EditRoles.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ExportModal.test.js delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/modals/ManageAccessModal.svelte delete mode 100644 packages/builder/src/components/backend/DataTable/modals/grid/GridCreateViewModal.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/BracketEdge.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/Controls.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/RoleNode.svelte create mode 100644 packages/builder/src/components/backend/RoleEditor/constants.js create mode 100644 packages/builder/src/components/backend/RoleEditor/utils.js delete mode 100644 packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte create mode 100644 packages/builder/src/components/common/AIFieldConfiguration.svelte create mode 100644 packages/builder/src/components/common/DetailPopover.svelte rename packages/{frontend-core/src/components/grid/controls => builder/src/components/common}/ToggleActionButtonGroup.svelte (82%) create mode 100644 packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/RowAction.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/data/roles.svelte rename packages/builder/src/pages/builder/app/[application]/data/{view/v2 => table/[tableId]}/[viewId]/_layout.svelte (95%) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte rename packages/builder/src/{components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte => pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte} (96%) rename packages/builder/src/{components/backend/TableNavigator/ViewNavItem => pages/builder/app/[application]/data/table/[tableId]/_components}/EditViewModal.svelte (87%) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte rename packages/builder/src/pages/builder/app/[application]/data/{view => table/[tableId]}/v1/[viewName]/_layout.svelte (100%) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte rename packages/builder/src/pages/builder/app/[application]/data/{view => table/[tableId]}/v1/index.svelte (100%) delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/utils.js delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/RolesTagsTableRenderer.svelte delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/TagsTableRenderer.svelte delete mode 100644 packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte create mode 100644 packages/builder/src/stores/builder/published.js create mode 100644 packages/builder/src/stores/builder/rowActions.js create mode 100644 packages/builder/src/stores/portal/featureFlags.js create mode 100644 packages/builder/src/templates/rowActions.js delete mode 100644 packages/client/src/components/app/dynamic-filter/FilterModal.svelte create mode 100644 packages/frontend-core/src/api/rowActions.js create mode 100644 packages/frontend-core/src/components/CoreFilterBuilder.svelte delete mode 100644 packages/frontend-core/src/components/FilterBuilder.svelte create mode 100644 packages/frontend-core/src/components/FilterField.svelte create mode 100644 packages/frontend-core/src/components/grid/cells/AICell.svelte create mode 100644 packages/frontend-core/src/components/grid/cells/RoleCell.svelte delete mode 100644 packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte delete mode 100644 packages/frontend-core/src/components/grid/controls/SizeButton.svelte delete mode 100644 packages/frontend-core/src/components/grid/controls/SortButton.svelte create mode 100644 packages/frontend-core/src/utils/schema.js delete mode 100644 packages/frontend-core/src/utils/theme.js create mode 100644 packages/server/src/api/routes/public/tests/Request.ts rename packages/server/src/api/routes/public/tests/{metrics.spec.js => metrics.spec.ts} (94%) create mode 100644 packages/server/src/api/routes/public/tests/security.spec.ts rename packages/server/src/api/routes/tests/{routing.spec.js => routing.spec.ts} (72%) rename packages/server/src/middleware/tests/{currentapp.spec.js => currentapp.spec.ts} (94%) rename packages/server/src/sdk/app/{rowActions.ts => rowActions/crud.ts} (80%) create mode 100644 packages/server/src/sdk/app/rowActions/index.ts create mode 100644 packages/server/src/sdk/app/rowActions/utils.ts create mode 100644 packages/server/src/sdk/app/views/utils.ts create mode 100644 packages/server/src/utilities/tests/utils.spec.ts create mode 100644 packages/shared-core/src/constants/ai.ts create mode 100644 packages/shared-core/src/constants/themes.ts create mode 100644 packages/shared-core/src/tests/themes.ts create mode 100644 packages/shared-core/src/themes.ts create mode 100644 packages/types/src/core/events.ts create mode 100644 packages/types/src/documents/app/theme.ts delete mode 100644 packages/types/src/documents/global/tenantInfo.ts delete mode 100644 packages/worker/src/api/controllers/global/tenant.ts delete mode 100644 packages/worker/src/api/routes/global/tenant.ts delete mode 100644 packages/worker/src/api/routes/global/tests/tenant.spec.ts diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4b9ebf1e5d..2d725bf28a 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -309,3 +309,27 @@ jobs: } else { console.log('All good, the submodule had been merged and setup correctly!') } + + check-lockfile: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + + - name: Use Node.js 20.x + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: yarn + - run: yarn install + - name: Check for yarn.lock changes + run: | + if [[ $(git status --porcelain) == *"yarn.lock"* ]]; then + echo "yarn.lock file needs to be modified. Please update it locally and commit the changes." + exit 1 + else + echo "yarn.lock file is unchanged." + fi diff --git a/.github/workflows/deploy-qa.yml b/.github/workflows/deploy-qa.yml index 2a30e44def..1339ad2eb9 100644 --- a/.github/workflows/deploy-qa.yml +++ b/.github/workflows/deploy-qa.yml @@ -3,7 +3,7 @@ name: Deploy QA on: push: branches: - - v3-ui + - master workflow_dispatch: jobs: diff --git a/.gitignore b/.gitignore index 32d1416f4a..bac643e5df 100644 --- a/.gitignore +++ b/.gitignore @@ -4,11 +4,10 @@ packages/server/runtime_apps/ .idea/ bb-airgapped.tar.gz *.iml - packages/server/build/oldClientVersions/**/* packages/builder/src/components/deploy/clientVersions.json - packages/server/src/integrations/tests/utils/*.lock +packages/builder/vite.config.mjs.timestamp* # Logs logs diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index a1230f3c37..e4858d4af0 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -22,7 +22,8 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json RUN ./scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production --frozen-lockfile +ARG TARGETPLATFORM +RUN --mount=type=cache,target=/root/.yarn/${TARGETPLATFORM} YARN_CACHE_FOLDER=/root/.yarn/${TARGETPLATFORM} yarn install --production --frozen-lockfile # copy the actual code COPY packages/server/dist packages/server/dist diff --git a/lerna.json b/lerna.json index df50828b30..2f9e66675b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.33.2", + "version": "3.0.0", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 8cd052ce82..9bef5d1656 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8 +Subproject commit 9bef5d1656b4f3c991447ded6d65b0eba393a140 diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index aee099e10a..e2fd975e40 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -28,6 +28,7 @@ export enum Config { OIDC = "oidc", OIDC_LOGOS = "logos_oidc", SCIM = "scim", + AI = "AI", } export const MIN_VALID_DATE = new Date(-2147483647000) diff --git a/packages/backend-core/src/context/identity.ts b/packages/backend-core/src/context/identity.ts index 84de3b68c9..41f4b1eb66 100644 --- a/packages/backend-core/src/context/identity.ts +++ b/packages/backend-core/src/context/identity.ts @@ -27,7 +27,7 @@ export function doInUserContext(user: User, ctx: Ctx, task: any) { hostInfo: { ipAddress: ctx.request.ip, // filled in by koa-useragent package - userAgent: ctx.userAgent._agent.source, + userAgent: ctx.userAgent.source, }, } return doInIdentityContext(userContext, task) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 4e93e8d9ee..4cb0a9c731 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -83,7 +83,7 @@ function getPackageJsonFields(): { if (isDev() && !isTest()) { try { const lerna = getParentFile("lerna.json") - localVersion = lerna.version + localVersion = `${lerna.version}+local` } catch { // } @@ -223,6 +223,8 @@ const environment = { BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, OPENAI_API_KEY: process.env.OPENAI_API_KEY, + MIN_VERSION_WITHOUT_POWER_ROLE: + process.env.MIN_VERSION_WITHOUT_POWER_ROLE || "3.0.0", } export function setEnv(newEnvVars: Partial): () => void { diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index c7bc1c817b..69bf7009b2 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -171,9 +171,9 @@ const identifyUser = async ( if (isSSOUser(user)) { providerType = user.providerType } - const accountHolder = account?.budibaseUserId === user._id || false - const verified = - account && account?.budibaseUserId === user._id ? account.verified : false + const accountHolder = await users.getExistingAccounts([user.email]) + const isAccountHolder = accountHolder.length > 0 + const verified = !!account && isAccountHolder && account.verified const installationId = await getInstallationId() const hosting = account ? account.hosting : getHostingFromEnv() const environment = getDeploymentEnvironment() @@ -185,7 +185,7 @@ const identifyUser = async ( installationId, tenantId, verified, - accountHolder, + accountHolder: isAccountHolder, providerType, builder, admin, @@ -207,9 +207,10 @@ const identifyAccount = async (account: Account) => { const environment = getDeploymentEnvironment() if (isCloudAccount(account)) { - if (account.budibaseUserId) { + const user = await users.getGlobalUserByEmail(account.email) + if (user?._id) { // use the budibase user as the id if set - id = account.budibaseUserId + id = user._id } } diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 85aa2293ef..b713f509e0 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -1,20 +1,26 @@ import { Cookie, Header } from "../constants" import { - getCookie, clearCookie, - openJwt, + getCookie, isValidInternalAPIKey, + openJwt, } from "../utils" import { getUser } from "../cache/user" import { getSession, updateSessionTTL } from "../security/sessions" import { buildMatcherRegex, matches } from "./matchers" -import { SEPARATOR, queryGlobalView, ViewName } from "../db" -import { getGlobalDB, doInTenant } from "../context" +import { queryGlobalView, SEPARATOR, ViewName } from "../db" +import { doInTenant, getGlobalDB } from "../context" import { decrypt } from "../security/encryption" import * as identity from "../context/identity" import env from "../environment" -import { Ctx, EndpointMatcher, SessionCookie, User } from "@budibase/types" -import { InvalidAPIKeyError, ErrorCode } from "../errors" +import { + Ctx, + EndpointMatcher, + LoginMethod, + SessionCookie, + User, +} from "@budibase/types" +import { ErrorCode, InvalidAPIKeyError } from "../errors" import tracer from "dd-trace" const ONE_MINUTE = env.SESSION_UPDATE_PERIOD @@ -26,16 +32,18 @@ interface FinaliseOpts { internal?: boolean publicEndpoint?: boolean version?: string - user?: any + user?: User | { tenantId: string } + loginMethod?: LoginMethod } function timeMinusOneMinute() { return new Date(Date.now() - ONE_MINUTE).toISOString() } -function finalise(ctx: any, opts: FinaliseOpts = {}) { +function finalise(ctx: Ctx, opts: FinaliseOpts = {}) { ctx.publicEndpoint = opts.publicEndpoint || false ctx.isAuthenticated = opts.authenticated || false + ctx.loginMethod = opts.loginMethod ctx.user = opts.user ctx.internal = opts.internal || false ctx.version = opts.version @@ -120,9 +128,10 @@ export default function ( } const tenantId = ctx.request.headers[Header.TENANT_ID] - let authenticated = false, - user = null, - internal = false + let authenticated: boolean = false, + user: User | { tenantId: string } | undefined = undefined, + internal: boolean = false, + loginMethod: LoginMethod | undefined = undefined if (authCookie && !apiKey) { const sessionId = authCookie.sessionId const userId = authCookie.userId @@ -146,6 +155,7 @@ export default function ( } // @ts-ignore user.csrfToken = session.csrfToken + loginMethod = LoginMethod.COOKIE if (session?.lastAccessedAt < timeMinusOneMinute()) { // make sure we denote that the session is still in use @@ -170,17 +180,16 @@ export default function ( apiKey, populateUser ) - if (valid && foundUser) { + if (valid) { authenticated = true + loginMethod = LoginMethod.API_KEY user = foundUser - } else if (valid) { - authenticated = true - internal = true + internal = !foundUser } } if (!user && tenantId) { user = { tenantId } - } else if (user) { + } else if (user && "password" in user) { delete user.password } // be explicit @@ -204,7 +213,14 @@ export default function ( } // isAuthenticated is a function, so use a variable to be able to check authed state - finalise(ctx, { authenticated, user, internal, version, publicEndpoint }) + finalise(ctx, { + authenticated, + user, + internal, + version, + publicEndpoint, + loginMethod, + }) if (isUser(user)) { return identity.doInUserContext(user, ctx, next) diff --git a/packages/backend-core/src/security/permissions.ts b/packages/backend-core/src/security/permissions.ts index 4ed2cd3954..929ae92909 100644 --- a/packages/backend-core/src/security/permissions.ts +++ b/packages/backend-core/src/security/permissions.ts @@ -1,4 +1,8 @@ -import { PermissionLevel, PermissionType } from "@budibase/types" +import { + PermissionLevel, + PermissionType, + BuiltinPermissionID, +} from "@budibase/types" import flatten from "lodash/flatten" import cloneDeep from "lodash/fp/cloneDeep" @@ -57,14 +61,6 @@ export function getAllowedLevels(userPermLevel: PermissionLevel): string[] { } } -export enum BuiltinPermissionID { - PUBLIC = "public", - READ_ONLY = "read_only", - WRITE = "write", - ADMIN = "admin", - POWER = "power", -} - export const BUILTIN_PERMISSIONS: { [key in keyof typeof BuiltinPermissionID]: { _id: (typeof BuiltinPermissionID)[key] diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index c14178cacb..4076be93a0 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,5 +1,4 @@ import semver from "semver" -import { BuiltinPermissionID, PermissionLevel } from "./permissions" import { prefixRoleID, getRoleParams, @@ -14,10 +13,13 @@ import { RoleUIMetadata, Database, App, + BuiltinPermissionID, + PermissionLevel, } from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" import { RoleColor, helpers } from "@budibase/shared-core" import { uniqBy } from "lodash" +import { default as env } from "../environment" export const BUILTIN_ROLE_IDS = { ADMIN: "ADMIN", @@ -50,7 +52,7 @@ export class Role implements RoleDoc { _id: string _rev?: string name: string - permissionId: string + permissionId: BuiltinPermissionID inherits?: string | string[] version?: string permissions: Record = {} @@ -59,7 +61,7 @@ export class Role implements RoleDoc { constructor( id: string, name: string, - permissionId: string, + permissionId: BuiltinPermissionID, uiMetadata?: RoleUIMetadata ) { this._id = id @@ -213,13 +215,32 @@ export function getBuiltinRole(roleId: string): Role | undefined { return cloneDeep(role) } +export function validInherits( + allRoles: RoleDoc[], + inherits?: string | string[] +): boolean { + if (!inherits) { + return false + } + const find = (id: string) => allRoles.find(r => roleIDsAreEqual(r._id!, id)) + if (Array.isArray(inherits)) { + const filtered = inherits.filter(roleId => find(roleId)) + return inherits.length !== 0 && filtered.length === inherits.length + } else { + return !!find(inherits) + } +} + /** * Works through the inheritance ranks to see how far up the builtin stack this ID is. */ export function builtinRoleToNumber(id: string) { const builtins = getBuiltinRoles() const MAX = Object.values(builtins).length + 1 - if (id === BUILTIN_IDS.ADMIN || id === BUILTIN_IDS.BUILDER) { + if ( + roleIDsAreEqual(id, BUILTIN_IDS.ADMIN) || + roleIDsAreEqual(id, BUILTIN_IDS.BUILDER) + ) { return MAX } let role = builtins[id], @@ -256,7 +277,9 @@ export async function roleToNumber(id: string) { // find the built-in roles, get their number, sort it, then get the last one const highestBuiltin: number | undefined = role.inherits .map(roleId => { - const foundRole = hierarchy.find(role => role._id === roleId) + const foundRole = hierarchy.find(role => + roleIDsAreEqual(role._id!, roleId) + ) if (foundRole) { return findNumber(foundRole) + 1 } @@ -290,7 +313,7 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string { : roleId1 } -export function compareRoleIds(roleId1: string, roleId2: string) { +export function roleIDsAreEqual(roleId1: string, roleId2: string) { // make sure both role IDs are prefixed correctly return prefixRoleID(roleId1) === prefixRoleID(roleId2) } @@ -323,7 +346,7 @@ export function findRole( roleId = prefixRoleID(roleId) } const dbRole = roles.find( - role => role._id && compareRoleIds(role._id, roleId) + role => role._id && roleIDsAreEqual(role._id, roleId) ) if (!dbRole && !isBuiltin(roleId) && opts?.defaultPublic) { return cloneDeep(BUILTIN_ROLES.PUBLIC) @@ -380,7 +403,7 @@ async function getAllUserRoles( ): Promise { const allRoles = await getAllRoles() // admins have access to all roles - if (userRoleId === BUILTIN_IDS.ADMIN) { + if (roleIDsAreEqual(userRoleId, BUILTIN_IDS.ADMIN)) { return allRoles } @@ -491,17 +514,21 @@ export async function getAllRoles(appId?: string): Promise { // need to combine builtin with any DB record of them (for sake of permissions) for (let builtinRoleId of externalBuiltinRoles) { const builtinRole = builtinRoles[builtinRoleId] - const dbBuiltin = roles.filter( - dbRole => - getExternalRoleID(dbRole._id!, dbRole.version) === builtinRoleId + const dbBuiltin = roles.filter(dbRole => + roleIDsAreEqual(dbRole._id!, builtinRoleId) )[0] if (dbBuiltin == null) { roles.push(builtinRole || builtinRoles.BASIC) } else { // remove role and all back after combining with the builtin roles = roles.filter(role => role._id !== dbBuiltin._id) - dbBuiltin._id = getExternalRoleID(dbBuiltin._id!, dbBuiltin.version) - roles.push(Object.assign(builtinRole, dbBuiltin)) + dbBuiltin._id = getExternalRoleID(builtinRole._id!, dbBuiltin.version) + roles.push({ + ...builtinRole, + ...dbBuiltin, + name: builtinRole.name, + _id: getExternalRoleID(builtinRole._id!, builtinRole.version), + }) } } // check permissions @@ -528,7 +555,10 @@ async function shouldIncludePowerRole(db: Database) { return true } - const isGreaterThan3x = semver.gte(creationVersion, "3.0.0") + const isGreaterThan3x = semver.gte( + creationVersion, + env.MIN_VERSION_WITHOUT_POWER_ROLE + ) return !isGreaterThan3x } @@ -544,9 +574,9 @@ export class AccessController { if ( tryingRoleId == null || tryingRoleId === "" || - tryingRoleId === userRoleId || - tryingRoleId === BUILTIN_IDS.BUILDER || - userRoleId === BUILTIN_IDS.BUILDER + roleIDsAreEqual(tryingRoleId, BUILTIN_IDS.BUILDER) || + roleIDsAreEqual(userRoleId!, tryingRoleId) || + roleIDsAreEqual(userRoleId!, BUILTIN_IDS.BUILDER) ) { return true } @@ -557,7 +587,7 @@ export class AccessController { } return ( - roleIds?.find(roleId => compareRoleIds(roleId, tryingRoleId)) !== + roleIds?.find(roleId => roleIDsAreEqual(roleId, tryingRoleId)) !== undefined ) } diff --git a/packages/backend-core/src/security/tests/permissions.spec.ts b/packages/backend-core/src/security/tests/permissions.spec.ts index 39348646fb..f98833c7cd 100644 --- a/packages/backend-core/src/security/tests/permissions.spec.ts +++ b/packages/backend-core/src/security/tests/permissions.spec.ts @@ -1,6 +1,7 @@ import cloneDeep from "lodash/cloneDeep" import * as permissions from "../permissions" import { BUILTIN_ROLE_IDS } from "../roles" +import { BuiltinPermissionID } from "@budibase/types" describe("levelToNumber", () => { it("should return 0 for EXECUTE", () => { @@ -77,7 +78,7 @@ describe("doesHaveBasePermission", () => { const rolesHierarchy = [ { roleId: BUILTIN_ROLE_IDS.ADMIN, - permissionId: permissions.BuiltinPermissionID.ADMIN, + permissionId: BuiltinPermissionID.ADMIN, }, ] expect( @@ -91,7 +92,7 @@ describe("doesHaveBasePermission", () => { const rolesHierarchy = [ { roleId: BUILTIN_ROLE_IDS.PUBLIC, - permissionId: permissions.BuiltinPermissionID.PUBLIC, + permissionId: BuiltinPermissionID.PUBLIC, }, ] expect( @@ -129,7 +130,7 @@ describe("getBuiltinPermissions", () => { describe("getBuiltinPermissionByID", () => { it("returns correct permission object for valid ID", () => { const expectedPermission = { - _id: permissions.BuiltinPermissionID.PUBLIC, + _id: BuiltinPermissionID.PUBLIC, name: "Public", permissions: [ new permissions.Permission( diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 949d7edf1b..e4b2b843af 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -13,6 +13,7 @@ import SqlTableQueryBuilder from "./sqlTable" import { Aggregation, AnySearchFilter, + ArrayFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, @@ -98,6 +99,23 @@ function isSqs(table: Table): boolean { ) } +function escapeQuotes(value: string, quoteChar = '"'): string { + return value.replace(new RegExp(quoteChar, "g"), `${quoteChar}${quoteChar}`) +} + +function wrap(value: string, quoteChar = '"'): string { + return `${quoteChar}${escapeQuotes(value, quoteChar)}${quoteChar}` +} + +function stringifyArray(value: any[], quoteStyle = '"'): string { + for (let i in value) { + if (typeof value[i] === "string") { + value[i] = wrap(value[i], quoteStyle) + } + } + return `[${value.join(",")}]` +} + const allowEmptyRelationships: Record = { [BasicOperator.EQUAL]: false, [BasicOperator.NOT_EQUAL]: true, @@ -152,30 +170,24 @@ class InternalBuilder { return this.query.meta.table } + get knexClient(): Knex.Client { + return this.knex.client as Knex.Client + } + getFieldSchema(key: string): FieldSchema | undefined { const { column } = this.splitter.run(key) return this.table.schema[column] } private quoteChars(): [string, string] { - switch (this.client) { - case SqlClient.ORACLE: - case SqlClient.POSTGRES: - return ['"', '"'] - case SqlClient.MS_SQL: - return ["[", "]"] - case SqlClient.MARIADB: - case SqlClient.MY_SQL: - case SqlClient.SQL_LITE: - return ["`", "`"] - } + const wrapped = this.knexClient.wrapIdentifier("foo", {}) + return [wrapped[0], wrapped[wrapped.length - 1]] } - // Takes a string like foo and returns a quoted string like [foo] for SQL Server - // and "foo" for Postgres. + // Takes a string like foo and returns a quoted string like [foo] for SQL + // Server and "foo" for Postgres. private quote(str: string): string { - const [start, end] = this.quoteChars() - return `${start}${str}${end}` + return this.knexClient.wrapIdentifier(str, {}) } private isQuoted(key: string): boolean { @@ -193,6 +205,52 @@ class InternalBuilder { return key.map(part => this.quote(part)).join(".") } + private quotedValue(value: string): string { + const formatter = this.knexClient.formatter(this.knexClient.queryBuilder()) + return formatter.wrap(value, false) + } + + private castIntToString(identifier: string | Knex.Raw): Knex.Raw { + switch (this.client) { + case SqlClient.ORACLE: { + return this.knex.raw("to_char(??)", [identifier]) + } + case SqlClient.POSTGRES: { + return this.knex.raw("??::TEXT", [identifier]) + } + case SqlClient.MY_SQL: + case SqlClient.MARIADB: { + return this.knex.raw("CAST(?? AS CHAR)", [identifier]) + } + case SqlClient.SQL_LITE: { + // Technically sqlite can actually represent numbers larger than a 64bit + // int as a string, but it does it using scientific notation (e.g. + // "1e+20") which is not what we want. Given that the external SQL + // databases are limited to supporting only 64bit ints, we settle for + // that here. + return this.knex.raw("printf('%d', ??)", [identifier]) + } + case SqlClient.MS_SQL: { + return this.knex.raw("CONVERT(NVARCHAR, ??)", [identifier]) + } + } + } + + // Unfortuantely we cannot rely on knex's identifier escaping because it trims + // the identifier string before escaping it, which breaks cases for us where + // columns that start or end with a space aren't referenced correctly anymore. + // + // So whenever you're using an identifier binding in knex, e.g. knex.raw("?? + // as ?", ["foo", "bar"]), you need to make sure you call this: + // + // knex.raw("?? as ?", [this.quotedIdentifier("foo"), "bar"]) + // + // Issue we filed against knex about this: + // https://github.com/knex/knex/issues/6143 + private rawQuotedIdentifier(key: string): Knex.Raw { + return this.knex.raw(this.quotedIdentifier(key)) + } + // Turns an identifier like a.b.c or `a`.`b`.`c` into ["a", "b", "c"] private splitIdentifier(key: string): string[] { const [start, end] = this.quoteChars() @@ -236,7 +294,7 @@ class InternalBuilder { const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { - return [this.knex.raw(`${this.quote(alias)}.*`)] + return [this.knex.raw("??", [`${alias}.*`])] } // get just the fields for this table return resource.fields @@ -258,30 +316,39 @@ class InternalBuilder { const columnSchema = schema[column] if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { - return this.knex.raw( - `${this.quotedIdentifier( - [table, column].join(".") - )}::money::numeric as ${this.quote(field)}` - ) + return this.knex.raw(`??::money::numeric as ??`, [ + this.rawQuotedIdentifier([table, column].join(".")), + this.knex.raw(this.quote(field)), + ]) } if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { // Time gets returned as timestamp from mssql, not matching the expected // HH:mm format - return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + + // TODO: figure out how to express this safely without string + // interpolation. + return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [ + this.rawQuotedIdentifier(field), + this.knex.raw(this.quote(field)), + ]) } - const quoted = table - ? `${this.quote(table)}.${this.quote(column)}` - : this.quote(field) - return this.knex.raw(quoted) + if (table) { + return this.rawQuotedIdentifier(`${table}.${column}`) + } else { + return this.rawQuotedIdentifier(field) + } }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, // so when we use them we need to wrap them in to_char(). This function // converts a field name to the appropriate identifier. - private convertClobs(field: string, opts?: { forSelect?: boolean }): string { + private convertClobs( + field: string, + opts?: { forSelect?: boolean } + ): Knex.Raw { if (this.client !== SqlClient.ORACLE) { throw new Error( "you've called convertClobs on a DB that's not Oracle, this is a mistake" @@ -290,7 +357,7 @@ class InternalBuilder { const parts = this.splitIdentifier(field) const col = parts.pop()! const schema = this.table.schema[col] - let identifier = this.quotedIdentifier(field) + let identifier = this.rawQuotedIdentifier(field) if ( schema.type === FieldType.STRING || @@ -301,9 +368,12 @@ class InternalBuilder { schema.type === FieldType.BARCODEQR ) { if (opts?.forSelect) { - identifier = `to_char(${identifier}) as ${this.quotedIdentifier(col)}` + identifier = this.knex.raw("to_char(??) as ??", [ + identifier, + this.rawQuotedIdentifier(col), + ]) } else { - identifier = `to_char(${identifier})` + identifier = this.knex.raw("to_char(??)", [identifier]) } } return identifier @@ -427,7 +497,6 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName @@ -449,8 +518,8 @@ class InternalBuilder { relationship.to && relationship.tableName ) { - const joinTable = mainKnex - .select(mainKnex.raw(1)) + const joinTable = this.knex + .select(this.knex.raw(1)) .from({ [toAlias]: relatedTableName }) let subQuery = joinTable.clone() const manyToMany = validateManyToMany(relationship) @@ -459,7 +528,7 @@ class InternalBuilder { if (!matchesTableName) { updatedKey = filterKey.replace( new RegExp(`^${relationship.column}.`), - `${aliases![relationship.tableName]}.` + `${aliases?.[relationship.tableName] || relationship.tableName}.` ) } else { updatedKey = filterKey @@ -485,9 +554,7 @@ class InternalBuilder { .where( `${throughAlias}.${manyToMany.from}`, "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) - ) + this.rawQuotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`) ) // in SQS the same junction table is used for different many-to-many relationships between the // two same tables, this is needed to avoid rows ending up in all columns @@ -516,7 +583,7 @@ class InternalBuilder { subQuery = subQuery.where( toKey, "=", - mainKnex.raw(this.quotedIdentifier(foreignKey)) + this.rawQuotedIdentifier(foreignKey) ) query = query.where(q => { @@ -546,7 +613,7 @@ class InternalBuilder { filters = this.parseFilters({ ...filters }) const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or - const allOr = filters.allOr + const shouldOr = filters.allOr const isSqlite = this.client === SqlClient.SQL_LITE const tableName = isSqlite ? this.table._id! : this.table.name @@ -610,7 +677,7 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - if (allOr) { + if (shouldOr) { query = query.or } query = builder.addRelationshipForFilter( @@ -626,85 +693,102 @@ class InternalBuilder { } const like = (q: Knex.QueryBuilder, key: string, value: any) => { - const fuzzyOr = filters?.fuzzyOr - const fnc = fuzzyOr || allOr ? "orWhere" : "where" - // postgres supports ilike, nothing else does - if (this.client === SqlClient.POSTGRES) { - return q[fnc](key, "ilike", `%${value}%`) - } else { - const rawFnc = `${fnc}Raw` - // @ts-ignore - return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + if (filters?.fuzzyOr || shouldOr) { + q = q.or + } + if ( + this.client === SqlClient.ORACLE || + this.client === SqlClient.SQL_LITE + ) { + return q.whereRaw(`LOWER(??) LIKE ?`, [ + this.rawQuotedIdentifier(key), `%${value.toLowerCase()}%`, ]) } + return q.whereILike( + // @ts-expect-error knex types are wrong, raw is fine here + this.rawQuotedIdentifier(key), + this.knex.raw("?", [`%${value}%`]) + ) } - const contains = (mode: AnySearchFilter, any: boolean = false) => { - const rawFnc = allOr ? "orWhereRaw" : "whereRaw" - const not = mode === filters?.notContains ? "NOT " : "" - function stringifyArray(value: Array, quoteStyle = '"'): string { - for (let i in value) { - if (typeof value[i] === "string") { - value[i] = `${quoteStyle}${value[i]}${quoteStyle}` - } + const contains = (mode: ArrayFilter, any = false) => { + function addModifiers(q: Knex.QueryBuilder) { + if (shouldOr || mode === filters?.containsAny) { + q = q.or } - return `[${value.join(",")}]` + if (mode === filters?.notContains) { + q = q.not + } + return q } + if (this.client === SqlClient.POSTGRES) { iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - const wrap = any ? "" : "'" - const op = any ? "\\?| array" : "@>" - const fieldNames = key.split(/\./g) - const table = fieldNames[0] - const col = fieldNames[1] - return q[rawFnc]( - `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( - value, - any ? "'" : '"' - )}${wrap}, FALSE)` - ) + q = addModifiers(q) + if (any) { + return q.whereRaw(`COALESCE(??::jsonb \\?| array??, FALSE)`, [ + this.rawQuotedIdentifier(key), + this.knex.raw(stringifyArray(value, "'")), + ]) + } else { + return q.whereRaw(`COALESCE(??::jsonb @> '??', FALSE)`, [ + this.rawQuotedIdentifier(key), + this.knex.raw(stringifyArray(value)), + ]) + } }) } else if ( this.client === SqlClient.MY_SQL || this.client === SqlClient.MARIADB ) { - const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - return q[rawFnc]( - `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( - value - )}'), FALSE)` - ) + return addModifiers(q).whereRaw(`COALESCE(?(??, ?), FALSE)`, [ + this.knex.raw(any ? "JSON_OVERLAPS" : "JSON_CONTAINS"), + this.rawQuotedIdentifier(key), + this.knex.raw(wrap(stringifyArray(value))), + ]) }) } else { - const andOr = mode === filters?.containsAny ? " OR " : " AND " iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { - let statement = "" - const identifier = this.quotedIdentifier(key) - for (let i in value) { - if (typeof value[i] === "string") { - value[i] = `%"${value[i].toLowerCase()}"%` - } else { - value[i] = `%${value[i]}%` - } - statement += `${ - statement ? andOr : "" - }COALESCE(LOWER(${identifier}), '') LIKE ?` - } - - if (statement === "") { + if (value.length === 0) { return q } - if (not) { - return q[rawFnc]( - `(NOT (${statement}) OR ${identifier} IS NULL)`, - value - ) - } else { - return q[rawFnc](statement, value) - } + q = q.where(subQuery => { + if (mode === filters?.notContains) { + subQuery = subQuery.not + } + + subQuery = subQuery.where(subSubQuery => { + for (const elem of value) { + if (mode === filters?.containsAny) { + subSubQuery = subSubQuery.or + } else { + subSubQuery = subSubQuery.and + } + + const lower = + typeof elem === "string" ? `"${elem.toLowerCase()}"` : elem + + subSubQuery = subSubQuery.whereLike( + // @ts-expect-error knex types are wrong, raw is fine here + this.knex.raw(`COALESCE(LOWER(??), '')`, [ + this.rawQuotedIdentifier(key), + ]), + `%${lower}%` + ) + } + }) + if (mode === filters?.notContains) { + subQuery = subQuery.or.whereNull( + // @ts-expect-error knex types are wrong, raw is fine here + this.rawQuotedIdentifier(key) + ) + } + return subQuery + }) + return q }) } } @@ -730,45 +814,46 @@ class InternalBuilder { } if (filters.oneOf) { - const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, ArrayOperator.ONE_OF, (q, key: string, array) => { - if (this.client === SqlClient.ORACLE) { - key = this.convertClobs(key) - array = Array.isArray(array) ? array : [array] - const binding = new Array(array.length).fill("?").join(",") - return q.whereRaw(`${key} IN (${binding})`, array) - } else { - return q[fnc](key, Array.isArray(array) ? array : [array]) + if (shouldOr) { + q = q.or } + if (this.client === SqlClient.ORACLE) { + // @ts-ignore + key = this.convertClobs(key) + } + return q.whereIn(key, Array.isArray(array) ? array : [array]) }, (q, key: string[], array) => { - if (this.client === SqlClient.ORACLE) { - const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})` - const binding = `(${array - .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`) - .join(",")})` - return q.whereRaw(`${keyStr} IN ${binding}`, array.flat()) - } else { - return q[fnc](key, Array.isArray(array) ? array : [array]) + if (shouldOr) { + q = q.or } + if (this.client === SqlClient.ORACLE) { + // @ts-ignore + key = key.map(k => this.convertClobs(k)) + } + return q.whereIn(key, Array.isArray(array) ? array : [array]) } ) } if (filters.string) { iterate(filters.string, BasicOperator.STRING, (q, key, value) => { - const fnc = allOr ? "orWhere" : "where" - // postgres supports ilike, nothing else does - if (this.client === SqlClient.POSTGRES) { - return q[fnc](key, "ilike", `${value}%`) - } else { - const rawFnc = `${fnc}Raw` - // @ts-ignore - return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + if (shouldOr) { + q = q.or + } + if ( + this.client === SqlClient.ORACLE || + this.client === SqlClient.SQL_LITE + ) { + return q.whereRaw(`LOWER(??) LIKE ?`, [ + this.rawQuotedIdentifier(key), `${value.toLowerCase()}%`, ]) + } else { + return q.whereILike(key, `${value}%`) } }) } @@ -795,67 +880,59 @@ class InternalBuilder { const schema = this.getFieldSchema(key) + let rawKey: string | Knex.Raw = key + let high = value.high + let low = value.low + if (this.client === SqlClient.ORACLE) { - // @ts-ignore - key = this.knex.raw(this.convertClobs(key)) + rawKey = this.convertClobs(key) + } else if ( + this.client === SqlClient.SQL_LITE && + schema?.type === FieldType.BIGINT + ) { + rawKey = this.knex.raw("CAST(?? AS INTEGER)", [ + this.rawQuotedIdentifier(key), + ]) + high = this.knex.raw("CAST(? AS INTEGER)", [value.high]) + low = this.knex.raw("CAST(? AS INTEGER)", [value.low]) + } + + if (shouldOr) { + q = q.or } if (lowValid && highValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw( - `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, - [value.low, value.high] - ) - } else { - const fnc = allOr ? "orWhereBetween" : "whereBetween" - return q[fnc](key, [value.low, value.high]) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.whereBetween(rawKey, [low, high]) } else if (lowValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ - value.low, - ]) - } else { - const fnc = allOr ? "orWhere" : "where" - return q[fnc](key, ">=", value.low) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.where(rawKey, ">=", low) } else if (highValid) { - if ( - schema?.type === FieldType.BIGINT && - this.client === SqlClient.SQL_LITE - ) { - return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ - value.high, - ]) - } else { - const fnc = allOr ? "orWhere" : "where" - return q[fnc](key, "<=", value.high) - } + // @ts-expect-error knex types are wrong, raw is fine here + return q.where(rawKey, "<=", high) } return q }) } if (filters.equal) { iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => { - const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (shouldOr) { + q = q.or + } if (this.client === SqlClient.MS_SQL) { - return q[fnc]( - `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, - [value] - ) - } else if (this.client === SqlClient.ORACLE) { - const identifier = this.convertClobs(key) - return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [ + return q.whereRaw(`CASE WHEN ?? = ? THEN 1 ELSE 0 END = 1`, [ + this.rawQuotedIdentifier(key), value, ]) + } else if (this.client === SqlClient.ORACLE) { + const identifier = this.convertClobs(key) + return q.where(subq => + // @ts-expect-error knex types are wrong, raw is fine here + subq.whereNotNull(identifier).andWhere(identifier, value) + ) } else { - return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [ + return q.whereRaw(`COALESCE(?? = ?, FALSE)`, [ + this.rawQuotedIdentifier(key), value, ]) } @@ -863,20 +940,30 @@ class InternalBuilder { } if (filters.notEqual) { iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => { - const fnc = allOr ? "orWhereRaw" : "whereRaw" + if (shouldOr) { + q = q.or + } if (this.client === SqlClient.MS_SQL) { - return q[fnc]( - `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, - [value] - ) + return q.whereRaw(`CASE WHEN ?? = ? THEN 1 ELSE 0 END = 0`, [ + this.rawQuotedIdentifier(key), + value, + ]) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - return q[fnc]( - `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, - [value] + return ( + q + .where(subq => + subq.not + // @ts-expect-error knex types are wrong, raw is fine here + .whereNull(identifier) + .and.where(identifier, "!=", value) + ) + // @ts-expect-error knex types are wrong, raw is fine here + .or.whereNull(identifier) ) } else { - return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [ + return q.whereRaw(`COALESCE(?? != ?, TRUE)`, [ + this.rawQuotedIdentifier(key), value, ]) } @@ -884,14 +971,18 @@ class InternalBuilder { } if (filters.empty) { iterate(filters.empty, BasicOperator.EMPTY, (q, key) => { - const fnc = allOr ? "orWhereNull" : "whereNull" - return q[fnc](key) + if (shouldOr) { + q = q.or + } + return q.whereNull(key) }) } if (filters.notEmpty) { iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => { - const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - return q[fnc](key) + if (shouldOr) { + q = q.or + } + return q.whereNotNull(key) }) } if (filters.contains) { @@ -976,9 +1067,7 @@ class InternalBuilder { const selectFields = qualifiedFields.map(field => this.convertClobs(field, { forSelect: true }) ) - query = query - .groupByRaw(groupByFields.join(", ")) - .select(this.knex.raw(selectFields.join(", "))) + query = query.groupBy(groupByFields).select(selectFields) } else { query = query.groupBy(qualifiedFields).select(qualifiedFields) } @@ -990,11 +1079,10 @@ class InternalBuilder { if (this.client === SqlClient.ORACLE) { const field = this.convertClobs(`${tableName}.${aggregation.field}`) query = query.select( - this.knex.raw( - `COUNT(DISTINCT ${field}) as ${this.quotedIdentifier( - aggregation.name - )}` - ) + this.knex.raw(`COUNT(DISTINCT ??) as ??`, [ + field, + aggregation.name, + ]) ) } else { query = query.countDistinct( @@ -1002,24 +1090,36 @@ class InternalBuilder { ) } } else { - query = query.count(`* as ${aggregation.name}`) + if (this.client === SqlClient.ORACLE) { + const field = this.convertClobs(`${tableName}.${aggregation.field}`) + query = query.select( + this.knex.raw(`COUNT(??) as ??`, [field, aggregation.name]) + ) + } else { + query = query.count(`${aggregation.field} as ${aggregation.name}`) + } } } else { - const field = `${tableName}.${aggregation.field} as ${aggregation.name}` - switch (op) { - case CalculationType.SUM: - query = query.sum(field) - break - case CalculationType.AVG: - query = query.avg(field) - break - case CalculationType.MIN: - query = query.min(field) - break - case CalculationType.MAX: - query = query.max(field) - break + const fieldSchema = this.getFieldSchema(aggregation.field) + if (!fieldSchema) { + // This should not happen in practice. + throw new Error( + `field schema missing for aggregation target: ${aggregation.field}` + ) } + + let aggregate = this.knex.raw("??(??)", [ + this.knex.raw(op), + this.rawQuotedIdentifier(`${tableName}.${aggregation.field}`), + ]) + + if (fieldSchema.type === FieldType.BIGINT) { + aggregate = this.castIntToString(aggregate) + } + + query = query.select( + this.knex.raw("?? as ??", [aggregate, aggregation.name]) + ) } } return query @@ -1059,9 +1159,11 @@ class InternalBuilder { } else { let composite = `${aliased}.${key}` if (this.client === SqlClient.ORACLE) { - query = query.orderByRaw( - `${this.convertClobs(composite)} ${direction} nulls ${nulls}` - ) + query = query.orderByRaw(`?? ?? nulls ??`, [ + this.convertClobs(composite), + this.knex.raw(direction), + this.knex.raw(nulls as string), + ]) } else { query = query.orderBy(composite, direction, nulls) } @@ -1091,17 +1193,22 @@ class InternalBuilder { private buildJsonField(field: string): string { const parts = field.split(".") - let tableField: string, unaliased: string + let unaliased: string + + let tableField: string if (parts.length > 1) { const alias = parts.shift()! unaliased = parts.join(".") - tableField = `${this.quote(alias)}.${this.quote(unaliased)}` + tableField = `${alias}.${unaliased}` } else { unaliased = parts.join(".") - tableField = this.quote(unaliased) + tableField = unaliased } + const separator = this.client === SqlClient.ORACLE ? " VALUE " : "," - return `'${unaliased}'${separator}${tableField}` + return this.knex + .raw(`?${separator}??`, [unaliased, this.rawQuotedIdentifier(tableField)]) + .toString() } maxFunctionParameters() { @@ -1197,13 +1304,13 @@ class InternalBuilder { subQuery = subQuery.where( correlatedTo, "=", - knex.raw(this.quotedIdentifier(correlatedFrom)) + this.rawQuotedIdentifier(correlatedFrom) ) - const standardWrap = (select: string): Knex.QueryBuilder => { + const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => { subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing - return knex.select(knex.raw(select)).from({ + return knex.select(select).from({ [toAlias]: subQuery, }) } @@ -1213,12 +1320,12 @@ class InternalBuilder { // need to check the junction table document is to the right column, this is just for SQS subQuery = this.addJoinFieldCheck(subQuery, relationship) wrapperQuery = standardWrap( - `json_group_array(json_object(${fieldList}))` + this.knex.raw(`json_group_array(json_object(${fieldList}))`) ) break case SqlClient.POSTGRES: wrapperQuery = standardWrap( - `json_agg(json_build_object(${fieldList}))` + this.knex.raw(`json_agg(json_build_object(${fieldList}))`) ) break case SqlClient.MARIADB: @@ -1232,21 +1339,25 @@ class InternalBuilder { case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( - `json_arrayagg(json_object(${fieldList}))` + this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) ) break - case SqlClient.MS_SQL: + case SqlClient.MS_SQL: { + const comparatorQuery = knex + .select(`${fromAlias}.*`) + // @ts-ignore - from alias syntax not TS supported + .from({ + [fromAlias]: subQuery + .select(`${toAlias}.*`) + .limit(getRelationshipLimit()), + }) + wrapperQuery = knex.raw( - `(SELECT ${this.quote(toAlias)} = (${knex - .select(`${fromAlias}.*`) - // @ts-ignore - from alias syntax not TS supported - .from({ - [fromAlias]: subQuery - .select(`${toAlias}.*`) - .limit(getRelationshipLimit()), - })} FOR JSON PATH))` + `(SELECT ?? = (${comparatorQuery} FOR JSON PATH))`, + [this.rawQuotedIdentifier(toAlias)] ) break + } default: throw new Error(`JSON relationships not implement for ${sqlClient}`) } @@ -1351,7 +1462,8 @@ class InternalBuilder { schema.constraints?.presence === true || schema.type === FieldType.FORMULA || schema.type === FieldType.AUTO || - schema.type === FieldType.LINK + schema.type === FieldType.LINK || + schema.type === FieldType.AI ) { continue } @@ -1473,7 +1585,7 @@ class InternalBuilder { query = this.addFilters(query, filters, { relationship: true }) // handle relationships with a CTE for all others - if (relationships?.length) { + if (relationships?.length && aggregations.length === 0) { const mainTable = this.query.tableAliases?.[this.query.endpoint.entityId] || this.query.endpoint.entityId @@ -1488,10 +1600,8 @@ class InternalBuilder { // add JSON aggregations attached to the CTE return this.addJsonRelationships(cte, tableName, relationships) } - // no relationships found - return query - else { - return query - } + + return query } update(opts: QueryOptions): Knex.QueryBuilder { diff --git a/packages/backend-core/src/tenancy/db.ts b/packages/backend-core/src/tenancy/db.ts index 332ecbca48..10477a8579 100644 --- a/packages/backend-core/src/tenancy/db.ts +++ b/packages/backend-core/src/tenancy/db.ts @@ -1,29 +1,6 @@ import { getDB } from "../db/db" import { getGlobalDBName } from "../context" -import { TenantInfo } from "@budibase/types" export function getTenantDB(tenantId: string) { return getDB(getGlobalDBName(tenantId)) } - -export async function saveTenantInfo(tenantInfo: TenantInfo) { - const db = getTenantDB(tenantInfo.tenantId) - // save the tenant info to db - return db.put({ - _id: "tenant_info", - ...tenantInfo, - }) -} - -export async function getTenantInfo( - tenantId: string -): Promise { - try { - const db = getTenantDB(tenantId) - const tenantInfo = (await db.get("tenant_info")) as TenantInfo - delete tenantInfo.owner.password - return tenantInfo - } catch { - return undefined - } -} diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index c96c615f4b..cbc0019303 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -16,14 +16,15 @@ import { isSSOUser, SaveUserOpts, User, - UserStatus, UserGroup, + UserIdentifier, + UserStatus, PlatformUserBySsoId, PlatformUserById, AnyDocument, } from "@budibase/types" import { - getAccountHolderFromUserIds, + getAccountHolderFromUsers, isAdmin, isCreator, validateUniqueUser, @@ -412,7 +413,9 @@ export class UserDB { ) } - static async bulkDelete(userIds: string[]): Promise { + static async bulkDelete( + users: Array + ): Promise { const db = getGlobalDB() const response: BulkUserDeleted = { @@ -421,13 +424,13 @@ export class UserDB { } // remove the account holder from the delete request if present - const account = await getAccountHolderFromUserIds(userIds) - if (account) { - userIds = userIds.filter(u => u !== account.budibaseUserId) + const accountHolder = await getAccountHolderFromUsers(users) + if (accountHolder) { + users = users.filter(u => u.userId !== accountHolder.userId) // mark user as unsuccessful response.unsuccessful.push({ - _id: account.budibaseUserId, - email: account.email, + _id: accountHolder.userId, + email: accountHolder.email, reason: "Account holder cannot be deleted", }) } @@ -435,7 +438,7 @@ export class UserDB { // Get users and delete const allDocsResponse = await db.allDocs({ include_docs: true, - keys: userIds, + keys: users.map(u => u.userId), }) const usersToDelete = allDocsResponse.rows.map(user => { return user.doc! diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index f4838597b6..0bff428fa9 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -70,6 +70,17 @@ export async function getAllUserIds() { return response.rows.map(row => row.id) } +export async function getAllUsers(): Promise { + const db = getGlobalDB() + const startKey = `${DocumentType.USER}${SEPARATOR}` + const response = await db.allDocs({ + startkey: startKey, + endkey: `${startKey}${UNICODE_MAX}`, + include_docs: true, + }) + return response.rows.map(row => row.doc) as User[] +} + export async function bulkUpdateGlobalUsers(users: User[]) { const db = getGlobalDB() return (await db.bulkDocs(users)) as BulkDocsResponse diff --git a/packages/backend-core/src/users/utils.ts b/packages/backend-core/src/users/utils.ts index e1e3da181d..91b667ce17 100644 --- a/packages/backend-core/src/users/utils.ts +++ b/packages/backend-core/src/users/utils.ts @@ -1,11 +1,9 @@ -import { CloudAccount, ContextUser, User, UserGroup } from "@budibase/types" +import { ContextUser, User, UserGroup, UserIdentifier } from "@budibase/types" import * as accountSdk from "../accounts" import env from "../environment" -import { getFirstPlatformUser } from "./lookup" +import { getExistingAccounts, getFirstPlatformUser } from "./lookup" import { EmailUnavailableError } from "../errors" -import { getTenantId } from "../context" import { sdk } from "@budibase/shared-core" -import { getAccountByTenantId } from "../accounts" import { BUILTIN_ROLE_IDS } from "../security/roles" import * as context from "../context" @@ -67,21 +65,17 @@ export async function validateUniqueUser(email: string, tenantId: string) { } /** - * For the given user id's, return the account holder if it is in the ids. + * For a list of users, return the account holder if there is an email match. */ -export async function getAccountHolderFromUserIds( - userIds: string[] -): Promise { +export async function getAccountHolderFromUsers( + users: Array +): Promise { if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - const tenantId = getTenantId() - const account = await getAccountByTenantId(tenantId) - if (!account) { - throw new Error(`Account not found for tenantId=${tenantId}`) - } - - const budibaseUserId = account.budibaseUserId - if (userIds.includes(budibaseUserId)) { - return account - } + const accountMetadata = await getExistingAccounts( + users.map(user => user.email) + ) + return users.find(user => + accountMetadata.map(metadata => metadata.email).includes(user.email) + ) } } diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index bc9a3b635c..5ba6fb36a1 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -102,6 +102,14 @@ export const useAppBuilders = () => { return useFeature(Feature.APP_BUILDERS) } +export const useBudibaseAI = () => { + return useFeature(Feature.BUDIBASE_AI) +} + +export const useAICustomConfigs = () => { + return useFeature(Feature.AI_CUSTOM_CONFIGS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index d3cec0f307..2401354fbb 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -1,15 +1,11 @@ - - (showTooltip = true)} on:mouseleave={() => (showTooltip = false)} on:focus={() => (showTooltip = true)} + {disabled} + style={accentStyle} > - - + {#if icon} + + {/if} + {#if $$slots} + + {/if} + {#if tooltip && showTooltip} +
+ +
+ {/if} + diff --git a/packages/bbui/src/ActionMenu/ActionMenu.svelte b/packages/bbui/src/ActionMenu/ActionMenu.svelte index 75ddd679da..f27854bc04 100644 --- a/packages/bbui/src/ActionMenu/ActionMenu.svelte +++ b/packages/bbui/src/ActionMenu/ActionMenu.svelte @@ -1,14 +1,20 @@ -
+
diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index 21635592d2..e95c7dd1b6 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -151,9 +151,9 @@ export default function positionDropdown(element, opts) { // Determine X strategy if (align === "right") { applyXStrategy(Strategies.EndToEnd) - } else if (align === "right-outside") { + } else if (align === "right-outside" || align === "right-context-menu") { applyXStrategy(Strategies.StartToEnd) - } else if (align === "left-outside") { + } else if (align === "left-outside" || align === "left-context-menu") { applyXStrategy(Strategies.EndToStart) } else if (align === "center") { applyXStrategy(Strategies.MidPoint) @@ -164,6 +164,12 @@ export default function positionDropdown(element, opts) { // Determine Y strategy if (align === "right-outside" || align === "left-outside") { applyYStrategy(Strategies.MidPoint) + } else if ( + align === "right-context-menu" || + align === "left-context-menu" + ) { + applyYStrategy(Strategies.StartToStart) + styles.top -= 5 // Manual adjustment for action menu padding } else { applyYStrategy(Strategies.StartToEnd) } @@ -240,7 +246,7 @@ export default function positionDropdown(element, opts) { } // Apply initial styles which don't need to change - element.style.position = "absolute" + element.style.position = "fixed" element.style.zIndex = "9999" // Set up a scroll listener diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte index 9e49d84d44..0a8917c3c1 100644 --- a/packages/bbui/src/Button/Button.svelte +++ b/packages/bbui/src/Button/Button.svelte @@ -17,6 +17,8 @@ export let tooltip = undefined export let newStyles = true export let id + export let ref + export let reverse = false const dispatch = createEventDispatcher() @@ -25,6 +27,7 @@ @@ -91,4 +97,11 @@ .spectrum-Button--secondary.new-styles.is-disabled { color: var(--spectrum-global-color-gray-500); } + .spectrum-Button .spectrum-Button-label + .spectrum-Icon { + margin-left: var(--spectrum-button-primary-icon-gap); + margin-right: calc( + -1 * (var(--spectrum-button-primary-textonly-padding-left-adjusted) - + var(--spectrum-button-primary-padding-left-adjusted)) + ); + } diff --git a/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte new file mode 100644 index 0000000000..d7aad5ccff --- /dev/null +++ b/packages/bbui/src/ButtonGroup/CollapsedButtonGroup.svelte @@ -0,0 +1,57 @@ + + + + + + {#each buttons as button} + handleClick(button)} disabled={button.disabled}> + {button.text || "Button"} + + {/each} + + diff --git a/packages/bbui/src/Form/Core/Switch.svelte b/packages/bbui/src/Form/Core/Switch.svelte index deffc19167..d7110a6e67 100644 --- a/packages/bbui/src/Form/Core/Switch.svelte +++ b/packages/bbui/src/Form/Core/Switch.svelte @@ -19,6 +19,7 @@ {disabled} on:change={onChange} on:click + on:click|stopPropagation {id} type="checkbox" class="spectrum-Switch-input" diff --git a/packages/bbui/src/Form/Core/TextField.svelte b/packages/bbui/src/Form/Core/TextField.svelte index 3335d3567b..917bb2a452 100644 --- a/packages/bbui/src/Form/Core/TextField.svelte +++ b/packages/bbui/src/Form/Core/TextField.svelte @@ -1,6 +1,6 @@ diff --git a/packages/bbui/src/Icon/Icon.svelte b/packages/bbui/src/Icon/Icon.svelte index 6ae1f4ca67..73ad8edd10 100644 --- a/packages/bbui/src/Icon/Icon.svelte +++ b/packages/bbui/src/Icon/Icon.svelte @@ -60,10 +60,11 @@ .newStyles { color: var(--spectrum-global-color-gray-700); } - + svg { + transition: color var(--spectrum-global-animation-duration-100, 130ms); + } svg.hoverable { pointer-events: all; - transition: color var(--spectrum-global-animation-duration-100, 130ms); } svg.hoverable:hover { color: var(--hover-color) !important; diff --git a/packages/bbui/src/InlineAlert/InlineAlert.svelte b/packages/bbui/src/InlineAlert/InlineAlert.svelte index 3b98936f62..edfa760eb8 100644 --- a/packages/bbui/src/InlineAlert/InlineAlert.svelte +++ b/packages/bbui/src/InlineAlert/InlineAlert.svelte @@ -8,6 +8,7 @@ export let onConfirm = undefined export let buttonText = "" export let cta = false + $: icon = selectIcon(type) // if newlines used, convert them to different elements $: split = message.split("\n") diff --git a/packages/bbui/src/List/ListItem.svelte b/packages/bbui/src/List/ListItem.svelte index 76b242cf9c..e979b2b684 100644 --- a/packages/bbui/src/List/ListItem.svelte +++ b/packages/bbui/src/List/ListItem.svelte @@ -1,55 +1,68 @@ - - -
-
- {#if icon} - + +
+ {#if icon === "StatusLight"} + + {:else if icon} +
+ +
{/if} - {#if avatar} - - {/if} - {#if title} - {title} - {/if} - {#if subtitle} - +
+ {#if title} +
+ {title} +
+ {/if} + {#if subtitle} +
+ {subtitle} +
+ {/if} +
+
+
+ + {#if showArrow} + {/if}
- {#if $$slots.default} -
- -
- {/if} -
+ diff --git a/packages/bbui/src/Menu/Item.svelte b/packages/bbui/src/Menu/Item.svelte index 05a33adda9..5e5f6d840c 100644 --- a/packages/bbui/src/Menu/Item.svelte +++ b/packages/bbui/src/Menu/Item.svelte @@ -27,7 +27,7 @@ const onClick = () => { if (actionMenu && !noClose) { - actionMenu.hide() + actionMenu.hideAll() } dispatch("click") } @@ -35,7 +35,7 @@
diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte index c88317c79f..873e769e21 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte @@ -3,7 +3,7 @@ automationStore, selectedAutomation, permissions, - selectedAutomationDisplayData, + tables, } from "stores/builder" import { Icon, @@ -17,6 +17,7 @@ AbsTooltip, InlineAlert, } from "@budibase/bbui" + import { sdk } from "@budibase/shared-core" import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import ActionModal from "./ActionModal.svelte" @@ -51,7 +52,12 @@ $: isAppAction && setPermissions(role) $: isAppAction && getPermissions(automationId) - $: triggerInfo = $selectedAutomationDisplayData?.triggerInfo + $: triggerInfo = sdk.automations.isRowAction($selectedAutomation) && { + title: "Automation trigger", + tableName: $tables.list.find( + x => x._id === $selectedAutomation.definition.trigger.inputs?.tableId + )?.name, + } async function setPermissions(role) { if (!role || !automationId) { @@ -187,10 +193,10 @@ {block} {webhookModal} /> - {#if isTrigger && triggerInfo} + {#if triggerInfo} {/if} {#if lastStep} diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte index 6e4d7c0099..4e9ca5fd53 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationNavItem.svelte @@ -17,11 +17,14 @@ let confirmDeleteDialog let updateAutomationDialog + $: isRowAction = sdk.automations.isRowAction(automation) + async function deleteAutomation() { try { await automationStore.actions.delete(automation) notifications.success("Automation deleted successfully") } catch (error) { + console.error(error) notifications.error("Error deleting automation") } } @@ -36,42 +39,7 @@ } const getContextMenuItems = () => { - const isRowAction = sdk.automations.isRowAction(automation) - const result = [] - if (!isRowAction) { - result.push( - ...[ - { - icon: "Delete", - name: "Delete", - keyBind: null, - visible: true, - disabled: false, - callback: confirmDeleteDialog.show, - }, - { - icon: "Edit", - name: "Edit", - keyBind: null, - visible: true, - disabled: !automation.definition.trigger, - callback: updateAutomationDialog.show, - }, - { - icon: "Duplicate", - name: "Duplicate", - keyBind: null, - visible: true, - disabled: - !automation.definition.trigger || - automation.definition.trigger?.name === "Webhook", - callback: duplicateAutomation, - }, - ] - ) - } - - result.push({ + const pause = { icon: automation.disabled ? "CheckmarkCircle" : "Cancel", name: automation.disabled ? "Activate" : "Pause", keyBind: null, @@ -83,8 +51,50 @@ automation.disabled ) }, - }) - return result + } + const del = { + icon: "Delete", + name: "Delete", + keyBind: null, + visible: true, + disabled: false, + callback: confirmDeleteDialog.show, + } + if (!isRowAction) { + return [ + { + icon: "Edit", + name: "Edit", + keyBind: null, + visible: true, + disabled: !automation.definition.trigger, + callback: updateAutomationDialog.show, + }, + { + icon: "Duplicate", + name: "Duplicate", + keyBind: null, + visible: true, + disabled: + !automation.definition.trigger || + automation.definition.trigger?.name === "Webhook", + callback: duplicateAutomation, + }, + pause, + del, + ] + } else { + return [ + { + icon: "Edit", + name: "Edit", + keyBind: null, + visible: true, + callback: updateAutomationDialog.show, + }, + del, + ] + } } const openContextMenu = e => { @@ -99,17 +109,17 @@ automationStore.actions.select(automation._id)} selectedBy={$userSelectedResourceMap[automation._id]} disabled={automation.disabled} > -
- -
+ {automation.name}? This action cannot be undone. - - + diff --git a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte index 58eebfdd3e..a26efdf243 100644 --- a/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte +++ b/packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte @@ -3,13 +3,21 @@ import { Modal, notifications, Layout } from "@budibase/bbui" import NavHeader from "components/common/NavHeader.svelte" import { onMount } from "svelte" - import { automationStore } from "stores/builder" + import { automationStore, tables } from "stores/builder" import AutomationNavItem from "./AutomationNavItem.svelte" + import { TriggerStepID } from "constants/backend/automations" export let modal export let webhookModal let searchString + const dsTriggers = [ + TriggerStepID.ROW_SAVED, + TriggerStepID.ROW_UPDATED, + TriggerStepID.ROW_DELETED, + TriggerStepID.ROW_ACTION, + ] + $: filteredAutomations = $automationStore.automations .filter(automation => { return ( @@ -17,31 +25,53 @@ automation.name.toLowerCase().includes(searchString.toLowerCase()) ) }) - .map(automation => ({ - ...automation, - displayName: - $automationStore.automationDisplayData[automation._id]?.displayName || - automation.name, - })) .sort((a, b) => { - const lowerA = a.displayName.toLowerCase() - const lowerB = b.displayName.toLowerCase() + const lowerA = a.name.toLowerCase() + const lowerB = b.name.toLowerCase() return lowerA > lowerB ? 1 : -1 }) - $: groupedAutomations = filteredAutomations.reduce((acc, auto) => { - const catName = auto.definition?.trigger?.event || "No Trigger" - acc[catName] ??= { - icon: auto.definition?.trigger?.icon || "AlertCircle", - name: (auto.definition?.trigger?.name || "No Trigger").toUpperCase(), - entries: [], - } - acc[catName].entries.push(auto) - return acc - }, {}) + $: groupedAutomations = groupAutomations(filteredAutomations) $: showNoResults = searchString && !filteredAutomations.length + const groupAutomations = automations => { + let groups = {} + + for (let auto of automations) { + let category = null + let dataTrigger = false + + // Group by datasource if possible + if (dsTriggers.includes(auto.definition?.trigger?.stepId)) { + if (auto.definition.trigger.inputs?.tableId) { + const tableId = auto.definition.trigger.inputs?.tableId + category = $tables.list.find(x => x._id === tableId)?.name + } + } + // Otherwise group by trigger + if (!category) { + category = auto.definition?.trigger?.name || "No Trigger" + } else { + dataTrigger = true + } + groups[category] ??= { + icon: auto.definition?.trigger?.icon || "AlertCircle", + name: category.toUpperCase(), + entries: [], + dataTrigger, + } + groups[category].entries.push(auto) + } + + return Object.values(groups).sort((a, b) => { + if (a.dataTrigger === b.dataTrigger) { + return a.name < b.name ? -1 : 1 + } + return a.dataTrigger ? -1 : 1 + }) + } + onMount(async () => { try { await automationStore.actions.fetch() @@ -88,16 +118,22 @@ diff --git a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte deleted file mode 100644 index 684cbd6cf4..0000000000 --- a/packages/builder/src/components/backend/DataTable/ViewDataTable.svelte +++ /dev/null @@ -1,80 +0,0 @@ - - -
- - - {#if view.calculation} - - {/if} - - - -
diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte deleted file mode 100644 index a35e1b034e..0000000000 --- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte +++ /dev/null @@ -1,58 +0,0 @@ - - -
- - - - - - - - - -
- - diff --git a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte deleted file mode 100644 index 87ca2fa142..0000000000 --- a/packages/builder/src/components/backend/DataTable/buttons/EditRolesButton.svelte +++ /dev/null @@ -1,13 +0,0 @@ - - - - Edit roles - - - - diff --git a/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte index 4fa1d07abd..e9352045ea 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/ExportButton.svelte @@ -1,20 +1,144 @@ - - Export - - - - + + + + Export + + + + {#if selectedRows?.length} + + + {selectedRows?.length} + {`row${selectedRows?.length > 1 ? "s" : ""} will be exported.`} + + + {:else} + + + Exporting all rows. + + + {/if} + + + + + + + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte new file mode 100644 index 0000000000..db446b3c9e --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridScreensButton.svelte @@ -0,0 +1,59 @@ + + + + + + Screens{screenCount ? `: ${screenCount}` : ""} + + + {#if !connectedScreens.length} + There aren't any screens connected to this data. + {:else} + The following screens are connected to this data. + + {#each connectedScreens as screen} + + {/each} + + {/if} +
+ +
+ diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte new file mode 100644 index 0000000000..148d8b1d5f --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSizeButton.svelte @@ -0,0 +1,127 @@ + + + + + + Size + + +
+ +
+ {#each rowSizeOptions as option} + changeRowHeight(option.size)} + > + {option.label} + + {/each} +
+
+
+ +
+ {#each columnSizeOptions as option} + columns.actions.changeAllColumnWidths(option.size)} + selected={option.selected} + > + {option.label} + + {/each} + {#if custom} + Custom + {/if} +
+
+ + + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte new file mode 100644 index 0000000000..244a1b1560 --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte @@ -0,0 +1,79 @@ + + + + + + Sort + + + + {/if} + diff --git a/packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte similarity index 100% rename from packages/builder/src/components/backend/DataTable/modals/grid/GridUsersTableButton.svelte rename to packages/builder/src/components/backend/DataTable/buttons/grid/GridUsersTableButton.svelte diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte new file mode 100644 index 0000000000..7b279d3948 --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte @@ -0,0 +1,267 @@ + + + + + + Configure calculations{count ? `: ${count}` : ""} + + + + {#if calculations.length} +
+ {#each calculations as calc, idx} + {idx === 0 ? "Calculate" : "and"} the + + deleteCalc(idx)} + color="var(--spectrum-global-color-gray-700)" + /> + {/each} + Group by +
+ +
+
+ {/if} +
+ = 5} + > + Add calculation + +
+ + +
+ +
+ + + diff --git a/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg b/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg new file mode 100644 index 0000000000..1702c9470a --- /dev/null +++ b/packages/builder/src/components/backend/DataTable/buttons/grid/magic-wand.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/packages/builder/src/components/backend/DataTable/formula.js b/packages/builder/src/components/backend/DataTable/formula.js index 7220a5ba4f..a9491abfef 100644 --- a/packages/builder/src/components/backend/DataTable/formula.js +++ b/packages/builder/src/components/backend/DataTable/formula.js @@ -8,6 +8,7 @@ const MAX_DEPTH = 1 const TYPES_TO_SKIP = [ FieldType.FORMULA, + FieldType.AI, FieldType.LONGFORM, FieldType.SIGNATURE_SINGLE, FieldType.ATTACHMENTS, diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a1bd54715b..d16bca3203 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -4,6 +4,7 @@ Button, Label, Select, + Multiselect, Toggle, Icon, DatePicker, @@ -25,6 +26,7 @@ import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" + import { featureFlags } from "stores/portal" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, @@ -34,6 +36,7 @@ } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils" import ConfirmDialog from "components/common/ConfirmDialog.svelte" + import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" @@ -49,18 +52,13 @@ import { isEnabled } from "helpers/featureFlags" import { getUserBindings } from "dataBinding" - const AUTO_TYPE = FieldType.AUTO - const FORMULA_TYPE = FieldType.FORMULA - const LINK_TYPE = FieldType.LINK - const STRING_TYPE = FieldType.STRING - const NUMBER_TYPE = FieldType.NUMBER - const JSON_TYPE = FieldType.JSON - const DATE_TYPE = FieldType.DATETIME + export let field const dispatch = createEventDispatcher() const { dispatch: gridDispatch, rows } = getContext("grid") - - export let field + const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}` + const SingleUserDefault = `{{ ${SafeID} }}` + const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}` let mounted = false let originalName @@ -103,13 +101,14 @@ let optionsValid = true $: rowGoldenSample = RowUtils.generateGoldenSample($rows) + $: aiEnabled = $featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } $: { // this parses any changes the user has made when creating a new internal relationship // into what we expect the schema to look like - if (editableColumn.type === LINK_TYPE) { + if (editableColumn.type === FieldType.LINK) { relationshipTableIdPrimary = table._id if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) { relationshipOpts2 = relationshipOpts2.filter( @@ -137,15 +136,16 @@ } $: initialiseField(field, savingColumn) $: checkConstraints(editableColumn) - $: required = hasDefault - ? false - : !!editableColumn?.constraints?.presence || primaryDisplay + $: required = + primaryDisplay || + editableColumn?.constraints?.presence === true || + editableColumn?.constraints?.presence?.allowEmpty === false $: uneditable = $tables.selected?._id === TableNames.USERS && UNEDITABLE_USER_FIELDS.includes(editableColumn.name) $: invalid = !editableColumn?.name || - (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) || + (editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) || Object.keys(errors).length !== 0 || !optionsValid $: errors = checkErrors(editableColumn) @@ -168,12 +168,12 @@ // used to select what different options can be displayed for column type $: canBeDisplay = canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn - $: canHaveDefault = - isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type) + $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES") + $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type) $: canBeRequired = - editableColumn?.type !== LINK_TYPE && + editableColumn?.type !== FieldType.LINK && !uneditable && - editableColumn?.type !== AUTO_TYPE && + editableColumn?.type !== FieldType.AUTO && !editableColumn.autocolumn $: hasDefault = editableColumn?.default != null && editableColumn?.default !== "" @@ -188,7 +188,6 @@ (originalName && SWITCHABLE_TYPES[field.type] && !editableColumn?.autocolumn) - $: allowedTypes = getAllowedTypes(datasource).map(t => ({ fieldId: makeFieldId(t.type, t.subtype), ...t, @@ -206,6 +205,11 @@ }, ...getUserBindings(), ] + $: sanitiseDefaultValue( + editableColumn.type, + editableColumn.constraints?.inclusion || [], + editableColumn.default + ) const fieldDefinitions = Object.values(FIELDS).reduce( // Storing the fields by complex field id @@ -218,7 +222,7 @@ function makeFieldId(type, subtype, autocolumn) { // don't make field IDs for auto types - if (type === AUTO_TYPE || autocolumn) { + if (type === FieldType.AUTO || autocolumn) { return type.toUpperCase() } else if ( type === FieldType.BB_REFERENCE || @@ -243,7 +247,7 @@ // Here we are setting the relationship values based on the editableColumn // This part of the code is used when viewing an existing field hence the check // for the tableId - if (editableColumn.type === LINK_TYPE && editableColumn.tableId) { + if (editableColumn.type === FieldType.LINK && editableColumn.tableId) { relationshipTableIdPrimary = table._id relationshipTableIdSecondary = editableColumn.tableId if (editableColumn.relationshipType in relationshipMap) { @@ -284,17 +288,33 @@ delete saveColumn.fieldId - if (saveColumn.type === AUTO_TYPE) { + if (saveColumn.type === FieldType.AUTO) { saveColumn = buildAutoColumn( $tables.selected.name, saveColumn.name, saveColumn.subtype ) } - if (saveColumn.type !== LINK_TYPE) { + if (saveColumn.type !== FieldType.LINK) { delete saveColumn.fieldName } + // Ensure we don't have a default value if we can't have one + if (!canHaveDefault || !defaultValuesEnabled) { + delete saveColumn.default + } + + // Ensure primary display columns are always required and don't have default values + if (primaryDisplay) { + saveColumn.constraints.presence = { allowEmpty: false } + delete saveColumn.default + } + + // Ensure the field is not required if we have a default value + if (saveColumn.default) { + saveColumn.constraints.presence = false + } + try { await tables.saveField({ originalName, @@ -362,9 +382,9 @@ editableColumn.subtype = definition.subtype // Default relationships many to many - if (editableColumn.type === LINK_TYPE) { + if (editableColumn.type === FieldType.LINK) { editableColumn.relationshipType = RelationshipType.MANY_TO_MANY - } else if (editableColumn.type === FORMULA_TYPE) { + } else if (editableColumn.type === FieldType.FORMULA) { editableColumn.formulaType = "dynamic" } } @@ -430,6 +450,7 @@ FIELDS.BOOLEAN, FIELDS.DATETIME, FIELDS.LINK, + ...(aiEnabled ? [FIELDS.AI] : []), FIELDS.LONGFORM, FIELDS.USER, FIELDS.USERS, @@ -483,17 +504,23 @@ fieldToCheck.constraints = {} } // some string types may have been built by server, may not always have constraints - if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) { + if ( + fieldToCheck.type === FieldType.STRING && + !fieldToCheck.constraints.length + ) { fieldToCheck.constraints.length = {} } // some number types made server-side will be missing constraints if ( - fieldToCheck.type === NUMBER_TYPE && + fieldToCheck.type === FieldType.NUMBER && !fieldToCheck.constraints.numericality ) { fieldToCheck.constraints.numericality = {} } - if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) { + if ( + fieldToCheck.type === FieldType.DATETIME && + !fieldToCheck.constraints.datetime + ) { fieldToCheck.constraints.datetime = {} } } @@ -541,6 +568,20 @@ return newError } + const sanitiseDefaultValue = (type, options, defaultValue) => { + if (!defaultValue?.length) { + return + } + // Delete default value for options fields if the option is no longer available + if (type === FieldType.OPTIONS && !options.includes(defaultValue)) { + delete editableColumn.default + } + // Filter array default values to only valid options + if (type === FieldType.ARRAY) { + editableColumn.default = defaultValue.filter(x => options.includes(x)) + } + } + onMount(() => { mounted = true }) @@ -554,13 +595,13 @@ on:input={e => { if ( !uneditable && - !(linkEditDisabled && editableColumn.type === LINK_TYPE) + !(linkEditDisabled && editableColumn.type === FieldType.LINK) ) { editableColumn.name = e.target.value } }} disabled={uneditable || - (linkEditDisabled && editableColumn.type === LINK_TYPE)} + (linkEditDisabled && editableColumn.type === FieldType.LINK)} error={errors?.name} /> {/if} @@ -574,7 +615,7 @@ getOptionValue={field => field.fieldId} getOptionIcon={field => field.icon} isOptionEnabled={option => { - if (option.type === AUTO_TYPE) { + if (option.type === FieldType.AUTO) { return availableAutoColumnKeys?.length > 0 } return true @@ -617,7 +658,7 @@ bind:optionColors={editableColumn.optionColors} bind:valid={optionsValid} /> - {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn} + {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
@@ -704,7 +745,7 @@ {tableOptions} {errors} /> - {:else if editableColumn.type === FORMULA_TYPE} + {:else if editableColumn.type === FieldType.FORMULA} {#if !externalTable}
@@ -747,12 +788,19 @@ />
- {:else if editableColumn.type === JSON_TYPE} - + {:else if editableColumn.type === FieldType.AI} + + {:else if editableColumn.type === FieldType.JSON} + {/if} - {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} + {#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn} role._id} - getOptionLabel={role => role.name} - /> - {#if selectedRole} - - x._id} - getOptionLabel={x => x.name} - disabled={shouldDisableRoleInput} - /> - {/if} -
- {#if !isCreating && !builtInRoles.includes(selectedRole.name)} - - {/if} -
- diff --git a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte b/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte deleted file mode 100644 index 2a31f5c452..0000000000 --- a/packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte +++ /dev/null @@ -1,224 +0,0 @@ - - - - {#if selectedRows?.length} - - - {selectedRows?.length} - {`row${selectedRows?.length > 1 ? "s" : ""} will be exported`} - - - {:else if appliedFilters?.length || (sorting?.sortOrder && sorting?.sortColumn)} - - {#if !appliedFilters} - - Exporting all rows - - {:else} - Filters applied - {/if} - - -
- - - {:else} - - - Exporting all rows - - - {/if} - - - - diff --git a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte index e0745c15a1..d7a98c564a 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte @@ -39,9 +39,7 @@ const selectTable = tableId => { tables.select(tableId) - if (!$isActive("./table/:tableId")) { - $goto(`./table/${tableId}`) - } + $goto(`./table/${tableId}`) } function openNode(datasource) { @@ -78,6 +76,13 @@ selectedBy={$userSelectedResourceMap[TableNames.USERS]} /> {/if} + $goto("./roles")} + selectedBy={$userSelectedResourceMap.roles} + /> {#each enrichedDataSources.filter(ds => ds.show) as datasource} + import { BaseEdge } from "@xyflow/svelte" + import { NodeWidth, GridResolution } from "./constants" + import { getContext } from "svelte" + + export let sourceX + export let sourceY + + const { bounds } = getContext("flow") + + $: bracketWidth = GridResolution * 3 + $: bracketHeight = $bounds.height / 2 + GridResolution * 2 + $: path = getCurlyBracePath( + sourceX + bracketWidth, + sourceY - bracketHeight, + sourceX + bracketWidth, + sourceY + bracketHeight + ) + + const getCurlyBracePath = (x1, y1, x2, y2) => { + const w = 2 // Thickness + const q = 1 // Intensity + const i = 28 // Inner radius strenth (lower is stronger) + const j = 32 // Outer radius strength (higher is stronger) + + // Calculate unit vector + var dx = x1 - x2 + var dy = y1 - y2 + var len = Math.sqrt(dx * dx + dy * dy) + dx = dx / len + dy = dy / len + + // Path control points + const qx1 = x1 + q * w * dy - j + const qy1 = y1 - q * w * dx + const qx2 = x1 - 0.25 * len * dx + (1 - q) * w * dy - i + const qy2 = y1 - 0.25 * len * dy - (1 - q) * w * dx + const tx1 = x1 - 0.5 * len * dx + w * dy - bracketWidth + const ty1 = y1 - 0.5 * len * dy - w * dx + const qx3 = x2 + q * w * dy - j + const qy3 = y2 - q * w * dx + const qx4 = x1 - 0.75 * len * dx + (1 - q) * w * dy - i + const qy4 = y1 - 0.75 * len * dy - (1 - q) * w * dx + + return `M ${x1} ${y1} Q ${qx1} ${qy1} ${qx2} ${qy2} T ${tx1} ${ty1} M ${x2} ${y2} Q ${qx3} ${qy3} ${qx4} ${qy4} T ${tx1} ${ty1}` + } + + + + + diff --git a/packages/builder/src/components/backend/RoleEditor/Controls.svelte b/packages/builder/src/components/backend/RoleEditor/Controls.svelte new file mode 100644 index 0000000000..b695a71a43 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/Controls.svelte @@ -0,0 +1,74 @@ + + +
+
+ flow.zoomIn({ duration: ZoomDuration })} + /> + flow.zoomOut({ duration: ZoomDuration })} + /> +
+ +
+
+ +
+ + diff --git a/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte b/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte new file mode 100644 index 0000000000..2c949ed0f9 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/EmptyStateNode.svelte @@ -0,0 +1,24 @@ + + +
+ Add custom roles for more granular control over permissions +
+ + diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte new file mode 100644 index 0000000000..84badeb6b2 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/RoleEdge.svelte @@ -0,0 +1,123 @@ + + + + + + + +
deleteEdge(id)} + on:mouseover={() => (iconHovered = true)} + on:mouseout={() => (iconHovered = false)} + > + +
+ + + diff --git a/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte new file mode 100644 index 0000000000..6169013d12 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/RoleEditor.svelte @@ -0,0 +1,8 @@ + + + + + diff --git a/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte new file mode 100644 index 0000000000..90c0a9fca8 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/RoleFlow.svelte @@ -0,0 +1,234 @@ + + +
+
+
+
+ dragging.set(true)} + onconnectend={() => dragging.set(false)} + onconnect={onConnect} + deleteKey={null} + > + + +
+ Manage roles +
+
Roles inherit permissions from each other
+ +
+ + diff --git a/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte new file mode 100644 index 0000000000..32a95f4278 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/RoleNode.svelte @@ -0,0 +1,231 @@ + + +
+
+
+
+
+ {data.displayName} +
+ {#if data.description} +
+ {data.description} +
+ {/if} +
+ {#if data.custom} +
+ + +
+ {/if} +
+ + +
+ + await deleteRole(id)} +/> + + + + (tempDisplayName = e.detail)} + /> + (tempDescription = e.detail)} + /> +
+ + (tempColor = e.detail)} /> +
+ + + + diff --git a/packages/builder/src/components/backend/RoleEditor/constants.js b/packages/builder/src/components/backend/RoleEditor/constants.js new file mode 100644 index 0000000000..6f188e2141 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/constants.js @@ -0,0 +1,9 @@ +export const ZoomDuration = 300 +export const MaxAutoZoom = 1.2 +export const GridResolution = 20 +export const NodeHeight = GridResolution * 3 +export const NodeWidth = GridResolution * 12 +export const NodeHSpacing = GridResolution * 6 +export const NodeVSpacing = GridResolution * 2 +export const MinHeight = GridResolution * 10 +export const EmptyStateID = "empty" diff --git a/packages/builder/src/components/backend/RoleEditor/utils.js b/packages/builder/src/components/backend/RoleEditor/utils.js new file mode 100644 index 0000000000..a958fc6401 --- /dev/null +++ b/packages/builder/src/components/backend/RoleEditor/utils.js @@ -0,0 +1,245 @@ +import dagre from "@dagrejs/dagre" +import { + NodeWidth, + NodeHeight, + GridResolution, + NodeHSpacing, + NodeVSpacing, + MinHeight, + EmptyStateID, +} from "./constants" +import { getNodesBounds, Position } from "@xyflow/svelte" +import { Roles } from "constants/backend" +import { roles } from "stores/builder" +import { get } from "svelte/store" + +// Calculates the bounds of all custom nodes +export const getBounds = nodes => { + const interactiveNodes = nodes.filter(node => node.data.interactive) + + // Empty state bounds which line up with bounds after adding first node + if (!interactiveNodes.length) { + return { + x: 0, + y: -3.5 * GridResolution, + width: 12 * GridResolution, + height: 10 * GridResolution, + } + } + let bounds = getNodesBounds(interactiveNodes) + + // Enforce a min size + if (bounds.height < MinHeight) { + const diff = MinHeight - bounds.height + bounds.height = MinHeight + bounds.y -= diff / 2 + } + return bounds +} + +// Gets the position of the basic role +export const getBasicPosition = bounds => ({ + x: bounds.x - NodeHSpacing - NodeWidth, + y: bounds.y + bounds.height / 2 - NodeHeight / 2, +}) + +// Gets the position of the admin role +export const getAdminPosition = bounds => ({ + x: bounds.x + bounds.width + NodeHSpacing, + y: bounds.y + bounds.height / 2 - NodeHeight / 2, +}) + +// Filters out invalid nodes and edges +const preProcessLayout = ({ nodes, edges }) => { + const ignoredIds = [Roles.PUBLIC, Roles.BASIC, Roles.ADMIN, EmptyStateID] + const targetlessIds = [Roles.POWER] + return { + nodes: nodes.filter(node => { + // Filter out ignored IDs + if (ignoredIds.includes(node.id)) { + return false + } + return true + }), + edges: edges.filter(edge => { + // Filter out edges from ignored IDs + if ( + ignoredIds.includes(edge.source) || + ignoredIds.includes(edge.target) + ) { + return false + } + // Filter out edges which have the same source and target + if (edge.source === edge.target) { + return false + } + // Filter out edges which target targetless roles + if (targetlessIds.includes(edge.target)) { + return false + } + return true + }), + } +} + +// Updates positions of nodes and edges into a nice graph structure +export const dagreLayout = ({ nodes, edges }) => { + const dagreGraph = new dagre.graphlib.Graph() + dagreGraph.setDefaultEdgeLabel(() => ({})) + dagreGraph.setGraph({ + rankdir: "LR", + ranksep: NodeHSpacing, + nodesep: NodeVSpacing, + }) + nodes.forEach(node => { + dagreGraph.setNode(node.id, { width: NodeWidth, height: NodeHeight }) + }) + edges.forEach(edge => { + dagreGraph.setEdge(edge.source, edge.target) + }) + dagre.layout(dagreGraph) + nodes.forEach(node => { + const pos = dagreGraph.node(node.id) + node.targetPosition = Position.Left + node.sourcePosition = Position.Right + node.position = { + x: Math.round((pos.x - NodeWidth / 2) / GridResolution) * GridResolution, + y: Math.round((pos.y - NodeHeight / 2) / GridResolution) * GridResolution, + } + }) + return { nodes, edges } +} + +const postProcessLayout = ({ nodes, edges }) => { + // Add basic and admin nodes at each edge + const bounds = getBounds(nodes) + const $roles = get(roles) + nodes.push({ + ...roleToNode($roles.find(role => role._id === Roles.BASIC)), + position: getBasicPosition(bounds), + }) + nodes.push({ + ...roleToNode($roles.find(role => role._id === Roles.ADMIN)), + position: getAdminPosition(bounds), + }) + + // Add custom edges for basic and admin brackets + edges.push({ + id: "basic-bracket", + source: Roles.BASIC, + target: Roles.ADMIN, + type: "bracket", + }) + edges.push({ + id: "admin-bracket", + source: Roles.ADMIN, + target: Roles.BASIC, + type: "bracket", + }) + + // Add empty state node if required + if (!nodes.some(node => node.data.interactive)) { + nodes.push({ + id: EmptyStateID, + type: "empty", + position: { + x: bounds.x + bounds.width / 2 - NodeWidth / 2, + y: bounds.y + bounds.height / 2 - NodeHeight / 2, + }, + data: {}, + measured: { + width: NodeWidth, + height: NodeHeight, + }, + deletable: false, + draggable: false, + connectable: false, + selectable: false, + }) + } + + return { nodes, edges } +} + +// Automatically lays out the graph, sanitising and enriching the structure +export const autoLayout = ({ nodes, edges }) => { + return postProcessLayout(dagreLayout(preProcessLayout({ nodes, edges }))) +} + +// Converts a role doc into a node structure +export const roleToNode = role => { + const custom = ![ + Roles.PUBLIC, + Roles.BASIC, + Roles.POWER, + Roles.ADMIN, + Roles.BUILDER, + ].includes(role._id) + const interactive = custom || role._id === Roles.POWER + return { + id: role._id, + sourcePosition: Position.Right, + targetPosition: Position.Left, + type: "role", + position: { x: 0, y: 0 }, + data: { + ...role.uiMetadata, + custom, + interactive, + }, + measured: { + width: NodeWidth, + height: NodeHeight, + }, + deletable: custom, + draggable: interactive, + connectable: interactive, + selectable: interactive, + } +} + +// Converts a node structure back into a role doc +export const nodeToRole = ({ node, edges }) => ({ + ...get(roles).find(role => role._id === node.id), + inherits: edges + .filter(x => x.target === node.id) + .map(x => x.source) + .concat(Roles.BASIC), + uiMetadata: { + displayName: node.data.displayName, + color: node.data.color, + description: node.data.description, + }, +}) + +// Builds a default layout from an array of roles +export const rolesToLayout = roles => { + let nodes = [] + let edges = [] + + // Add all nodes and edges + for (let role of roles) { + // Add node for this role + nodes.push(roleToNode(role)) + + // Add edges for this role + let inherits = [] + if (role.inherits) { + inherits = Array.isArray(role.inherits) ? role.inherits : [role.inherits] + } + for (let sourceRole of inherits) { + if (!roles.some(x => x._id === sourceRole)) { + continue + } + edges.push({ + id: `${sourceRole}-${role._id}`, + source: sourceRole, + target: role._id, + }) + } + } + return { + nodes, + edges, + } +} diff --git a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte index f1d85a6a30..ce966d0daa 100644 --- a/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte +++ b/packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte @@ -4,7 +4,7 @@ BBReferenceFieldSubType, SourceName, } from "@budibase/types" - import { Select, Toggle, Multiselect } from "@budibase/bbui" + import { Select, Toggle, Multiselect, Label, Layout } from "@budibase/bbui" import { DB_TYPE_INTERNAL } from "constants/backend" import { API } from "api" import { parseFile } from "./utils" @@ -140,84 +140,91 @@ } -
- - -
-{#if fileName && Object.keys(validation).length === 0} -

No valid fields, try another file

-{:else if rows.length > 0 && !error} -
- {#each Object.keys(validation) as name} -
- {name} - - {:else} -

Rows will be updated based on the table's primary key.

+ +
+ + + {#if fileName && Object.keys(validation).length === 0} +
No valid fields - please try another file.
+ {:else if fileName && rows.length > 0 && !error} +
+ {#each Object.keys(validation) as name} +
+ {name} + - -
-{#if rawRows.length > 0 && !error} -
- {#each Object.entries(schema) as [name, column]} -
- {column.name} - + +
+ + + {#if rawRows.length > 0 && !error} +
+ {#each Object.entries(schema) as [name, column]} +
+ {column.name} + -
-{/if} + {/if} + diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte index 03da9f3fd3..c0e030845f 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableNavItem/DeleteConfirmationModal.svelte @@ -104,7 +104,7 @@
{/if} -

Please enter the app name below to confirm.

+

Please enter the table name below to confirm.

diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte index ab79a8fff0..6b64096e2e 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableNavItem/TableNavItem.svelte @@ -20,14 +20,6 @@ const getContextMenuItems = () => { return [ - { - icon: "Delete", - name: "Delete", - keyBind: null, - visible: true, - disabled: false, - callback: deleteConfirmationModal.show, - }, { icon: "Edit", name: "Edit", @@ -36,6 +28,14 @@ disabled: false, callback: editModal.show, }, + { + icon: "Delete", + name: "Delete", + keyBind: null, + visible: true, + disabled: false, + callback: deleteConfirmationModal.show, + }, ] } diff --git a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte index f21230d7a6..f97bd2487b 100644 --- a/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte +++ b/packages/builder/src/components/backend/TableNavigator/TableNavigator.svelte @@ -1,33 +1,15 @@
{#each sortedTables as table, idx} selectTable(table._id)} /> - {#each [...Object.entries(table.views || {})].sort() as [name, view], idx (idx)} - { - if (view.version === 2) { - $goto(`./view/v2/${encodeURIComponent(view.id)}`) - } else { - $goto(`./view/v1/${encodeURIComponent(name)}`) - } - }} - /> - {/each} {/each}
diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte b/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte deleted file mode 100644 index ba30cea165..0000000000 --- a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/ViewNavItem.svelte +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte index b62c8af03d..5596ade573 100644 --- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte +++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte @@ -1,13 +1,7 @@ + + + {/if} + {/each} +{/if} diff --git a/packages/builder/src/components/common/DetailPopover.svelte b/packages/builder/src/components/common/DetailPopover.svelte new file mode 100644 index 0000000000..f1e81a6340 --- /dev/null +++ b/packages/builder/src/components/common/DetailPopover.svelte @@ -0,0 +1,77 @@ + + + + +
+ +
+ + +
+
+
+ {title} +
+ +
+
+ +
+
+ + + diff --git a/packages/builder/src/components/common/RoleIcon.svelte b/packages/builder/src/components/common/RoleIcon.svelte index 1bd6ba49bc..3b48935e0c 100644 --- a/packages/builder/src/components/common/RoleIcon.svelte +++ b/packages/builder/src/components/common/RoleIcon.svelte @@ -1,12 +1,14 @@ diff --git a/packages/builder/src/components/common/RoleSelect.svelte b/packages/builder/src/components/common/RoleSelect.svelte index 6006b8ab8d..38b84e964d 100644 --- a/packages/builder/src/components/common/RoleSelect.svelte +++ b/packages/builder/src/components/common/RoleSelect.svelte @@ -3,7 +3,7 @@ import { roles } from "stores/builder" import { licensing } from "stores/portal" - import { Constants, RoleUtils } from "@budibase/frontend-core" + import { Constants } from "@budibase/frontend-core" import { createEventDispatcher } from "svelte" import { capitalise } from "helpers" @@ -49,7 +49,8 @@ let options = roles .filter(role => allowedRoles.includes(role._id)) .map(role => ({ - name: enrichLabel(role.name), + color: role.uiMetadata.color, + name: enrichLabel(role.uiMetadata.displayName), _id: role._id, })) if (allowedRoles.includes(Constants.Roles.CREATOR)) { @@ -64,7 +65,8 @@ // Allow all core roles let options = roles.map(role => ({ - name: enrichLabel(role.name), + color: role.uiMetadata.color, + name: enrichLabel(role.uiMetadata.displayName), _id: role._id, })) @@ -100,7 +102,7 @@ if (role._id === Constants.Roles.CREATOR || role._id === RemoveID) { return null } - return RoleUtils.getRoleColour(role._id) + return role.color || "var(--spectrum-global-color-static-magenta-400)" } const getIcon = role => { diff --git a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte b/packages/builder/src/components/common/ToggleActionButtonGroup.svelte similarity index 82% rename from packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte rename to packages/builder/src/components/common/ToggleActionButtonGroup.svelte index 497e77c2c9..8a5778534f 100644 --- a/packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte +++ b/packages/builder/src/components/common/ToggleActionButtonGroup.svelte @@ -9,7 +9,7 @@ export let options -
+
{#each options as option} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte index c23edbeb58..819ed10880 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte @@ -33,7 +33,7 @@ const getSchemaFields = resourceId => { const { schema } = getSchemaForDatasourcePlus(resourceId) - return Object.values(schema || {}) + return Object.values(schema || {}).filter(field => !field.readonly) } const onFieldsChanged = e => { diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js index 606ee41d02..b171b34111 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js @@ -25,3 +25,4 @@ export { default as OpenModal } from "./OpenModal.svelte" export { default as CloseModal } from "./CloseModal.svelte" export { default as ClearRowSelection } from "./ClearRowSelection.svelte" export { default as DownloadFile } from "./DownloadFile.svelte" +export { default as RowAction } from "./RowAction.svelte" diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json index 4022926e7f..631e3119e8 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json @@ -178,6 +178,11 @@ "name": "Download File", "type": "data", "component": "DownloadFile" + }, + { + "name": "Row Action", + "type": "data", + "component": "RowAction" } ] } diff --git a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte index db2289345f..6f3a13a745 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte @@ -2,10 +2,11 @@ import DraggableList from "../DraggableList/DraggableList.svelte" import ButtonSetting from "./ButtonSetting.svelte" import { createEventDispatcher } from "svelte" - import { Helpers } from "@budibase/bbui" + import { Helpers, Menu, MenuItem, Popover } from "@budibase/bbui" import { componentStore } from "stores/builder" import { getEventContextBindings } from "dataBinding" import { cloneDeep, isEqual } from "lodash/fp" + import { getRowActionButtonTemplates } from "templates/rowActions" export let componentInstance export let componentBindings @@ -17,13 +18,14 @@ const dispatch = createEventDispatcher() - let focusItem let cachedValue + let rowActionTemplates = [] + let anchor + let popover $: if (!isEqual(value, cachedValue)) { cachedValue = cloneDeep(value) } - $: buttonList = sanitizeValue(cachedValue) || [] $: buttonCount = buttonList.length $: eventContextBindings = getEventContextBindings({ @@ -73,17 +75,32 @@ _instanceName: Helpers.uuid(), text: cfg.text, type: cfg.type || "primary", - }, - {} + } ) } - const addButton = () => { + const addCustomButton = () => { const newButton = buildPseudoInstance({ text: `Button ${buttonCount + 1}`, }) dispatch("change", [...buttonList, newButton]) - focusItem = newButton._id + popover.hide() + } + + const addRowActionTemplate = template => { + dispatch("change", [...buttonList, template]) + popover.hide() + } + + const addButton = async () => { + rowActionTemplates = await getRowActionButtonTemplates({ + component: componentInstance, + }) + if (rowActionTemplates.length) { + popover.show() + } else { + addCustomButton() + } } const removeButton = id => { @@ -105,12 +122,11 @@ listItemKey={"_id"} listType={ButtonSetting} listTypeProps={itemProps} - focus={focusItem} draggable={buttonCount > 1} /> {/if} -
+ + + Custom button + {#each rowActionTemplates as template} + addRowActionTemplate(template)}> + {template.text} + + {/each} + + + diff --git a/packages/builder/src/pages/builder/app/[application]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/_layout.svelte index be8337b376..8e109d3145 100644 --- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte @@ -228,7 +228,7 @@ .top-nav { flex: 0 0 60px; background: var(--background); - padding-left: var(--spacing-xl); + padding: 0 var(--spacing-xl); display: grid; grid-template-columns: 1fr auto 1fr; flex-direction: row; @@ -269,6 +269,7 @@ flex-direction: row; justify-content: flex-end; align-items: center; + margin-right: calc(-1 * var(--spacing-xl)); } .toprightnav :global(.avatars) { diff --git a/packages/builder/src/pages/builder/app/[application]/data/roles.svelte b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte new file mode 100644 index 0000000000..0177577730 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/roles.svelte @@ -0,0 +1,8 @@ + + + diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte similarity index 95% rename from packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte index 3d0cffc387..9c1baa723e 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/_layout.svelte @@ -12,7 +12,7 @@ stateKey: "selectedViewId", validate: id => $viewsV2.list?.some(view => view.id === id), update: viewsV2.select, - fallbackUrl: "../../", + fallbackUrl: "../", store: viewsV2, routify, decode: decodeURIComponent, diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte new file mode 100644 index 0000000000..39b1dc4768 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte @@ -0,0 +1,78 @@ + + + + + + {#if calculation} + + {/if} + + + + {#if !calculation} + + + generateButton?.show()} /> + {/if} + + + + diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte new file mode 100644 index 0000000000..ecc85e7622 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/CreateViewButton.svelte @@ -0,0 +1,133 @@ + + + (name = null)} + width={540} +> + + {#if firstView} + + {:else} +
+ +
+ {/if} + +
+
+ (calculation = false)} + selected={!calculation} + icon="Rail" + /> +
+
+ (calculation = true)} + selected={calculation} + icon="123" + /> +
+
+ +
+ +
+ + + diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte similarity index 96% rename from packages/builder/src/components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte index 774a2f987a..0b5f18b70b 100644 --- a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/DeleteConfirmationModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/DeleteViewModal.svelte @@ -20,6 +20,7 @@ } notifications.success("View deleted") } catch (error) { + console.error(error) notifications.error("Error deleting view") } } diff --git a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/EditViewModal.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte similarity index 87% rename from packages/builder/src/components/backend/TableNavigator/ViewNavItem/EditViewModal.svelte rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte index 0809d55884..0f39fa063d 100644 --- a/packages/builder/src/components/backend/TableNavigator/ViewNavItem/EditViewModal.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/EditViewModal.svelte @@ -39,7 +39,7 @@ - - + + diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte new file mode 100644 index 0000000000..bf3d073f60 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte @@ -0,0 +1,384 @@ + + +
+ + + + +
+ {table?._id === TableNames.USERS ? "App users" : table?.name || ""} +
+ {#if tableSelectedBy} + + {/if} + {#if tableEditable} + + {/if} + + {#if hasViews} +
+ {#each v1Views as view (view.name)} + {@const selectedBy = $userSelectedResourceMap[view.name]} + openViewContextMenu(e, view)} + data-id={view.name} + > +
+ {view.name} +
+ {#if selectedBy} + + {/if} + openViewContextMenu(e, view)} + hoverable + name="MoreSmallList" + color="var(--spectrum-global-color-gray-600)" + hoverColor="var(--spectrum-global-color-gray-900)" + /> + + {/each} + {#each views as view (view.id)} + {@const selectedBy = $userSelectedResourceMap[view.id]} + openViewContextMenu(e, view)} + data-id={view.id} + > +
+ {view.name} +
+ {#if selectedBy} + + {/if} + openViewContextMenu(e, view)} + hoverable + name="MoreSmallList" + color="var(--spectrum-global-color-gray-600)" + hoverColor="var(--spectrum-global-color-gray-900)" + /> + + {/each} +
+ {/if} + {#if !hasViews && tableEditable} + + + To create subsets of data, control access and more, create a view. + + {/if} + {#if overflowedViews.length} + +
+ + {overflowedViews.length} more + +
+ {#each overflowedViews as view} + + + editOverflowView(view)}> + Edit + + deleteOverflowView(view)}> + Delete + + + {/each} + + {/if} + {#if hasViews} + + {/if} +
+ +{#if table && tableEditable} + + +{/if} + +{#if editableView} + + +{/if} + + diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte index 8c60dbdd69..da05196c04 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_layout.svelte @@ -3,6 +3,7 @@ import { tables, builderStore } from "stores/builder" import * as routify from "@roxi/routify" import { onDestroy } from "svelte" + import ViewNavBar from "./_components/ViewNavBar.svelte" $: tableId = $tables.selectedTableId $: builderStore.selectResource(tableId) @@ -20,4 +21,17 @@ onDestroy(stopSyncing) - +
+ + +
+ + diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte index d79a0bc0ad..5684e77e31 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte @@ -1,7 +1,97 @@ {#if $tables?.selected?.name} @@ -40,7 +119,56 @@
{/if} - + + + + {#if isUsersTable && $appStore.features.disableUserMetadata} + + {/if} + + {#if relationshipsEnabled} + + {/if} + {#if !isUsersTable} + + + + generateButton?.show()} /> + generateButton?.show()} /> + + {/if} + + + + + + + + + + + + {#if isUsersTable} + + {:else} + + {/if} + {:else} Create your first table to start building {/if} diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte deleted file mode 100644 index 39dbcb9d11..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/[field]/index.svelte +++ /dev/null @@ -1,10 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte deleted file mode 100644 index 348ed0b5bf..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/[rowId]/index.svelte +++ /dev/null @@ -1,7 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte deleted file mode 100644 index cecec0ab53..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/relationship/index.svelte +++ /dev/null @@ -1,7 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/_layout.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/_layout.svelte similarity index 100% rename from packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/_layout.svelte rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/_layout.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte new file mode 100644 index 0000000000..2c822569b7 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/[viewName]/index.svelte @@ -0,0 +1,91 @@ + + +
+ {#if view} +
+ + + {#if view.calculation} + + {/if} + + + +
+ {:else}Create your first table to start building{/if} +
+ + diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/index.svelte similarity index 100% rename from packages/builder/src/pages/builder/app/[application]/data/view/v1/index.svelte rename to packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/v1/index.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte deleted file mode 100644 index 623cd224db..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/view/index.svelte +++ /dev/null @@ -1,19 +0,0 @@ - diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte deleted file mode 100644 index 51149b602d..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/view/v1/[viewName]/index.svelte +++ /dev/null @@ -1,18 +0,0 @@ - - -{#if selectedView} - -{:else}Create your first table to start building{/if} - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte deleted file mode 100644 index c2281710ba..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/[viewId]/index.svelte +++ /dev/null @@ -1,5 +0,0 @@ - - - diff --git a/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte deleted file mode 100644 index c11ca87023..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/view/v2/index.svelte +++ /dev/null @@ -1,5 +0,0 @@ - diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte index d5a696c6bf..d40f28e65a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte @@ -8,6 +8,7 @@ import InfoDisplay from "./InfoDisplay.svelte" import analytics, { Events } from "analytics" import { shouldDisplaySetting } from "@budibase/frontend-core" + import { getContext, setContext } from "svelte" export let componentDefinition export let componentInstance @@ -19,6 +20,16 @@ export let includeHidden = false export let tag + // Sometimes we render component settings using a complicated nested + // component instance technique. This results in instances with IDs that + // don't exist anywhere in the tree. Therefore we need to keep track of + // what the real component tree ID is so we can always find it. + const rootId = getContext("rootId") + if (!rootId) { + setContext("rootId", componentInstance._id) + } + $: componentInstance._rootId = rootId || componentInstance._id + $: sections = getSections( componentInstance, componentDefinition, diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte index 03bf771beb..93044cdb9a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte @@ -4,9 +4,12 @@ export let title export let body export let icon = "HelpOutline" + export let quiet = false + export let warning = false + export let error = false -
+
{#if title}
@@ -16,7 +19,7 @@ {@html body} {:else} - + {@html body} @@ -24,6 +27,23 @@
diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte deleted file mode 100644 index 9ad41ad652..0000000000 --- a/packages/builder/src/pages/builder/portal/users/users/_components/UpdateRolesModal.svelte +++ /dev/null @@ -1,74 +0,0 @@ - - - - - Update {user.email}'s role for {app.name}. - - opt.label} + getOptionValue={opt => opt.value} + on:change={e => { + handleFilterChange({ + logicalOperator: e.detail, + }) + }} + placeholder={false} + /> + + of the following filter groups: +
+ {/if} + {#if editableFilters?.groups?.length} +
+ {#each editableFilters?.groups as group, groupIdx} +
+
+
+ + {getGroupPrefix(groupIdx, editableFilters.logicalOperator)} + + + { + const updated = { ...filter, field: e.detail } + onFieldChange(updated) + onFilterFieldUpdate(updated, groupIdx, filterIdx) + }} + placeholder="Column" + /> + + opt.label} + getOptionValue={opt => opt.value} + on:change={e => { + handleFilterChange({ + onEmptyFilter: e.detail, + }) + }} + placeholder={false} + /> + + when all filters are empty +
+ {/if} +
+ + + + +
+ +
+ {:else} + None of the table column can be used for filtering. + {/if} + +
+ + diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte deleted file mode 100644 index 3a0c789b9e..0000000000 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ /dev/null @@ -1,379 +0,0 @@ - - -
- - {#if fieldOptions?.length} - - {#if !fieldFilters?.length} - Add your first filter expression. - {:else} - - {#if behaviourFilters} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => handleOnEmptyFilter(e.detail)} - placeholder={null} - /> - {/if} -
- {/if} - {/if} - - {#if fieldFilters?.length} -
- {#if filtersLabel} -
- -
- {/if} -
- {#each fieldFilters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - {#if allowBindings} - - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
-
- {/if} -
- -
- {:else} - None of the table column can be used for filtering. - {/if} - -
- - diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte new file mode 100644 index 0000000000..2f034ddf3e --- /dev/null +++ b/packages/frontend-core/src/components/FilterField.svelte @@ -0,0 +1,319 @@ + + +
+ + + + + + +
+
+ {#if filter.valueType === FilterValueType.BINDING} + + {:else} +
+ {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} + + {:else} + + {/if} +
+ {/if} +
+ +
+ + {#if !disabled && allowBindings && filter.field && !filter.noValue} + + +
{ + bindingDrawer.show() + }} + > + +
+ {/if} +
+
+
+ + diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte index 489426df1e..4640561afd 100644 --- a/packages/frontend-core/src/components/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -27,7 +27,8 @@
option.email} diff --git a/packages/frontend-core/src/components/grid/cells/AICell.svelte b/packages/frontend-core/src/components/grid/cells/AICell.svelte new file mode 100644 index 0000000000..38e81cefd3 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/AICell.svelte @@ -0,0 +1,99 @@ + + + + +
+
+ {value || ""} +
+
+ +{#if isOpen} + +