From a6f6942288f674a902de77220d030e8cb4d81f6a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 16:10:07 +0100 Subject: [PATCH 01/20] Fixing an issue with corrupt relationship records referencing rows which don't exist, this is a temporary measure as these relationships should be cleaned up correctly but for now ignore any which reference rows which no longer exist. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 12 +++++++---- .../src/db/linkedRows/LinkController.ts | 20 ++++++++++--------- packages/types/src/sdk/db.ts | 2 +- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 274c1b9e93..472e0f44ad 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -213,17 +213,21 @@ export class DatabaseImpl implements Database { async getMultiple( ids: string[], - opts?: { allowMissing?: boolean } + opts?: { allowMissing?: boolean; excludeDocs?: boolean } ): Promise { // get unique ids = [...new Set(ids)] + const includeDocs = !opts?.excludeDocs const response = await this.allDocs({ keys: ids, - include_docs: true, + include_docs: includeDocs, }) const rowUnavailable = (row: RowResponse) => { // row is deleted - key lookup can return this - if (row.doc == null || ("deleted" in row.value && row.value.deleted)) { + if ( + (includeDocs && row.doc == null) || + (row.value && "deleted" in row.value && row.value.deleted) + ) { return true } return row.error === "not_found" @@ -237,7 +241,7 @@ export class DatabaseImpl implements Database { const missingIds = missing.map(row => row.key).join(", ") throw new Error(`Unable to get documents: ${missingIds}`) } - return rows.map(row => row.doc!) + return rows.map(row => (includeDocs ? row.doc! : row.value)) } async remove(idOrDoc: string | Document, rev?: string) { diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 85a160713b..3c0601c9ee 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -211,19 +211,21 @@ class LinkController { linkedSchema?.type === FieldType.LINK && linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY ) { - let links = ( - await getLinkDocuments({ - tableId: field.tableId, - rowId: linkId, - }) - ).filter( - link => - link.id !== row._id && link.fieldName === linkedSchema.name + let links = await getLinkDocuments({ + tableId: field.tableId, + rowId: linkId, + fieldName: linkedSchema.name, + }) + + // check all the related rows exist + const foundRecords = await this._db.getMultiple( + links.map(l => l.id), + { allowMissing: true, excludeDocs: true } ) // The 1 side of 1:N is already related to something else // You must remove the existing relationship - if (links.length > 0) { + if (foundRecords.length > 0) { throw new Error( `1:N Relationship Error: Record already linked to another.` ) diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index a081f4f1a2..49baaa5bb1 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -133,7 +133,7 @@ export interface Database { exists(docId: string): Promise getMultiple( ids: string[], - opts?: { allowMissing?: boolean } + opts?: { allowMissing?: boolean; excludeDocs?: boolean } ): Promise remove(idOrDoc: Document): Promise remove(idOrDoc: string, rev?: string): Promise From 86846eff3f79192385f420b200b1a58b59a0042f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 16:22:32 +0100 Subject: [PATCH 02/20] Small fix. --- packages/server/src/db/linkedRows/LinkController.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 3c0601c9ee..944542e73f 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -214,8 +214,10 @@ class LinkController { let links = await getLinkDocuments({ tableId: field.tableId, rowId: linkId, - fieldName: linkedSchema.name, - }) + }).filter( + link => + link.id !== row._id && link.fieldName === linkedSchema.name + ) // check all the related rows exist const foundRecords = await this._db.getMultiple( From 6fb844753b4ffcc7b3af857c723a0f193b2bc1f5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 16:24:54 +0100 Subject: [PATCH 03/20] Another small fix. --- packages/server/src/db/linkedRows/LinkController.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/db/linkedRows/LinkController.ts b/packages/server/src/db/linkedRows/LinkController.ts index 944542e73f..1cd4240d4b 100644 --- a/packages/server/src/db/linkedRows/LinkController.ts +++ b/packages/server/src/db/linkedRows/LinkController.ts @@ -211,10 +211,12 @@ class LinkController { linkedSchema?.type === FieldType.LINK && linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY ) { - let links = await getLinkDocuments({ - tableId: field.tableId, - rowId: linkId, - }).filter( + let links = ( + await getLinkDocuments({ + tableId: field.tableId, + rowId: linkId, + }) + ).filter( link => link.id !== row._id && link.fieldName === linkedSchema.name ) From 586dd8fea74726963177af3118806fe4eafd6288 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 18:10:58 +0100 Subject: [PATCH 04/20] Fixing an issue with newlines between coalesce statements in Postgres - we were escaping newlines even if they were valid when given a list of JSON operations to perform. --- .../routes/tests/queries/generic-sql.spec.ts | 29 +++++++++++++++++++ packages/server/src/integrations/postgres.ts | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index e6da4693eb..0979f8bed3 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -28,6 +28,7 @@ describe.each( const config = setup.getConfig() const isOracle = dbName === DatabaseName.ORACLE const isMsSQL = dbName === DatabaseName.SQL_SERVER + const isPostgres = dbName === DatabaseName.POSTGRES let rawDatasource: Datasource let datasource: Datasource @@ -47,6 +48,9 @@ describe.each( transformer: "return data", readable: true, } + if (query.fields?.sql && typeof query.fields.sql !== "string") { + throw new Error("Unable to create with knex structure in 'sql' field") + } return await config.api.query.save( { ...defaultQuery, ...query }, expectations @@ -207,6 +211,31 @@ describe.each( expect(prodQuery.parameters).toBeUndefined() expect(prodQuery.schema).toBeDefined() }) + + isPostgres && + it("should be able to handle a JSON aggregate with newlines", async () => { + const jsonStatement = `COALESCE(json_build_object('name', name),'{"name":{}}'::json)` + const query = await createQuery({ + fields: { + sql: client("test_table") + .select([ + "*", + client.raw( + `${jsonStatement} as json,\n${jsonStatement} as json2` + ), + ]) + .toString(), + }, + }) + const res = await config.api.query.execute( + query._id!, + {}, + { + status: 200, + } + ) + expect(res).toBeDefined() + }) }) }) diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 3652864991..28400f616f 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -41,7 +41,7 @@ if (types) { types.setTypeParser(1184, (val: any) => val) // timestampz } -const JSON_REGEX = /'{.*}'::json/s +const JSON_REGEX = /'{\s*.*?\s*}'::json/gs const Sql = sql.Sql interface PostgresConfig { From 0c532721877395c9f81be8bf26319c8c059c3597 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 9 Oct 2024 08:22:54 +0100 Subject: [PATCH 05/20] Allow empty queries when searching tables --- packages/frontend-core/src/api/tables.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/api/tables.js b/packages/frontend-core/src/api/tables.js index 34d2371e1a..823e1d1675 100644 --- a/packages/frontend-core/src/api/tables.js +++ b/packages/frontend-core/src/api/tables.js @@ -40,7 +40,7 @@ export const buildTableEndpoints = API => ({ sortType, paginate, }) => { - if (!tableId || !query) { + if (!tableId) { return { rows: [], } From bf62153b8b28b4688e4ee777c312561d881a9cfb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:16:32 +0200 Subject: [PATCH 06/20] Add test with conditions --- .../src/api/routes/global/tests/users.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index a654c42359..c8e71f7eb4 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -741,6 +741,25 @@ describe("/api/global/users", () => { it("should throw an error if public query performed", async () => { await config.api.users.searchUsers({}, { status: 403, noHeaders: true }) }) + + it("should be able to search using logical conditions", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { + $and: { + conditions: [ + { + $and: { + conditions: [{ string: { email: user.email } }], + }, + }, + ], + }, + }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0].email).toBe(user.email) + }) }) describe("DELETE /api/global/users/:userId", () => { From 0e24df2ddf125c1c9d11d20273712007163a2797 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:32:12 +0200 Subject: [PATCH 07/20] Improve types --- packages/shared-core/src/filters.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index b10375acb0..40c70a8a23 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -639,19 +639,19 @@ export function fixupFilterArrays(filters: SearchFilters) { return filters } -export function search( - docs: Record[], - query: RowSearchParams -): SearchResponse> { +export function search>( + docs: T[], + query: Omit +): SearchResponse { let result = runQuery(docs, query.query) if (query.sort) { result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) } - let totalRows = result.length + const totalRows = result.length if (query.limit) { result = limit(result, query.limit.toString()) } - const response: SearchResponse> = { rows: result } + const response: SearchResponse = { rows: result } if (query.countRows) { response.totalRows = totalRows } From 8b0c84b2ea67aa65a19f85b539d3e518bacd1aaf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:32:39 +0200 Subject: [PATCH 08/20] In memory filters --- packages/backend-core/src/users/users.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index d8546afa8b..96fb79c0d0 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -24,6 +24,7 @@ import * as context from "../context" import { getGlobalDB } from "../context" import { isCreator } from "./utils" import { UserDB } from "./db" +import { dataFilters } from "@budibase/shared-core" type GetOpts = { cleanup?: boolean } @@ -263,9 +264,12 @@ export async function paginatedUsers({ cleanup: true, }) } else { - // no search, query allDocs - const response = await db.allDocs(getGlobalUserParams(null, opts)) - userList = response.rows.map((row: any) => row.doc) + const response = await db.allDocs(getGlobalUserParams(null, opts)) + userList = response.rows.map(row => row.doc!) + + if (query) { + userList = dataFilters.search(userList, { query }).rows + } } return pagination(userList, pageSize, { paginate: true, From 3cf96c589cd102958529d88c2b806f238aa3cd4d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:33:15 +0200 Subject: [PATCH 09/20] Implement logical conditions --- packages/shared-core/src/utils.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 14b3c84425..4847e911e9 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -5,6 +5,7 @@ import { SearchFilters, BasicOperator, ArrayOperator, + isLogicalSearchOperator, } from "@budibase/types" import * as Constants from "./constants" import { removeKeyNumbering } from "./filters" @@ -97,10 +98,20 @@ export function isSupportedUserSearch(query: SearchFilters) { { op: BasicOperator.EQUAL, key: "_id" }, { op: ArrayOperator.ONE_OF, key: "_id" }, ] - for (let [key, operation] of Object.entries(query)) { + for (const [key, operation] of Object.entries(query)) { if (typeof operation !== "object") { return false } + + if (isLogicalSearchOperator(key)) { + for (const condition of query[key]!.conditions) { + if (!isSupportedUserSearch(condition)) { + return false + } + } + return true + } + const fields = Object.keys(operation || {}) // this filter doesn't contain options - ignore if (fields.length === 0) { From d811b9527f82ce55f1bae7c5c0a4a3bbfdeee84b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:44:04 +0200 Subject: [PATCH 10/20] Fix limit issues --- packages/backend-core/src/users/users.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 96fb79c0d0..f4838597b6 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -263,13 +263,17 @@ export async function paginatedUsers({ userList = await bulkGetGlobalUsersById(query?.oneOf?._id, { cleanup: true, }) + } else if (query) { + // TODO: this should use SQS search, but the logic is built in the 'server' package. Using the in-memory filtering to get this working meanwhile + const response = await db.allDocs( + getGlobalUserParams(null, { ...opts, limit: undefined }) + ) + userList = response.rows.map(row => row.doc!) + userList = dataFilters.search(userList, { query, limit: opts.limit }).rows } else { + // no search, query allDocs const response = await db.allDocs(getGlobalUserParams(null, opts)) userList = response.rows.map(row => row.doc!) - - if (query) { - userList = dataFilters.search(userList, { query }).rows - } } return pagination(userList, pageSize, { paginate: true, From c2a5f673aeb4df00ce3f31cf14cb7594fc1bc926 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 10:51:51 +0200 Subject: [PATCH 11/20] Lint --- packages/server/src/api/routes/tests/search.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 51965b5574..1ccc9bfdc9 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -187,7 +187,6 @@ describe.each([ if (isInMemory) { return dataFilters.search(_.cloneDeep(rows), { ...this.query, - tableId: tableOrViewId, }) } else { return config.api.row.search(tableOrViewId, this.query) From abd7b3b84e5e1fda05e4987e34cc66072d33e347 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 9 Oct 2024 09:12:59 +0000 Subject: [PATCH 12/20] Bump version to 2.32.13 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 55721b6b2d..faed02052e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.12", + "version": "2.32.13", "npmClient": "yarn", "packages": [ "packages/*", From 9d06c705ac47af317c3acb41d4b36a0236b9ee82 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 12:47:43 +0200 Subject: [PATCH 13/20] Remove ContextUser usages in favor of just ids --- .../src/api/controllers/table/internal.ts | 4 ++-- .../api/controllers/table/tests/utils.spec.ts | 2 +- .../server/src/api/controllers/table/utils.ts | 20 +++++++++---------- .../src/sdk/app/tables/internal/index.ts | 5 ++--- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 4286d51d3e..256c5b1f0a 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -33,7 +33,7 @@ export async function save( try { const { table } = await sdk.tables.internal.save(tableToSave, { - user: ctx.user, + userId: ctx.user._id, rowsToImport: rows, tableId: ctx.request.body._id, renaming, @@ -72,7 +72,7 @@ export async function bulkImport( await handleDataImport(table, { importRows: rows, identifierFields, - user: ctx.user, + userId: ctx.user._id, }) return table } diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts index dad0146696..68b00d3268 100644 --- a/packages/server/src/api/controllers/table/tests/utils.spec.ts +++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts @@ -41,7 +41,7 @@ describe("utils", () => { const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }] - const result = await importToRows(data, table, config.user) + const result = await importToRows(data, table, config.user?._id) expect(result).toEqual([ expect.objectContaining({ autoId: 1, diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index d568e5f33e..106d0e23a6 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -18,7 +18,6 @@ import { quotas } from "@budibase/pro" import { events, context, features } from "@budibase/backend-core" import { AutoFieldSubType, - ContextUser, Datasource, Row, SourceName, @@ -122,7 +121,7 @@ export function makeSureTableUpToDate(table: Table, tableToSave: Table) { export async function importToRows( data: Row[], table: Table, - user?: ContextUser, + userId?: string, opts?: { keepCouchId: boolean } ) { const originalTable = table @@ -136,7 +135,7 @@ export async function importToRows( // We use a reference to table here and update it after input processing, // so that we can auto increment auto IDs in imported data properly - const processed = await inputProcessing(user?._id, table, row, { + const processed = await inputProcessing(userId, table, row, { noAutoRelationships: true, }) row = processed @@ -167,11 +166,10 @@ export async function importToRows( export async function handleDataImport( table: Table, - opts?: { identifierFields?: string[]; user?: ContextUser; importRows?: Row[] } + opts?: { identifierFields?: string[]; userId?: string; importRows?: Row[] } ) { const schema = table.schema const identifierFields = opts?.identifierFields || [] - const user = opts?.user const importRows = opts?.importRows if (!importRows || !isRows(importRows) || !isSchema(schema)) { @@ -181,7 +179,7 @@ export async function handleDataImport( const db = context.getAppDB() const data = parse(importRows, table) - const finalData = await importToRows(data, table, user, { + const finalData = await importToRows(data, table, opts?.userId, { keepCouchId: identifierFields.includes("_id"), }) @@ -282,22 +280,22 @@ export function checkStaticTables(table: Table) { class TableSaveFunctions { db: Database - user?: ContextUser + userId?: string oldTable?: Table importRows?: Row[] rows: Row[] constructor({ - user, + userId, oldTable, importRows, }: { - user?: ContextUser + userId?: string oldTable?: Table importRows?: Row[] }) { this.db = context.getAppDB() - this.user = user + this.userId = userId this.oldTable = oldTable this.importRows = importRows // any rows that need updated @@ -329,7 +327,7 @@ class TableSaveFunctions { table = await handleSearchIndexes(table) table = await handleDataImport(table, { importRows: this.importRows, - user: this.user, + userId: this.userId, }) if (await features.flags.isEnabled("SQS")) { await sdk.tables.sqs.addTable(table) diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index c0beed0db8..7c72834ee0 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -5,7 +5,6 @@ import { ViewStatisticsSchema, ViewV2, Row, - ContextUser, } from "@budibase/types" import { hasTypeChanged, @@ -27,7 +26,7 @@ import { quotas } from "@budibase/pro" export async function save( table: Table, opts?: { - user?: ContextUser + userId?: string tableId?: string rowsToImport?: Row[] renaming?: RenameColumn @@ -63,7 +62,7 @@ export async function save( // saving a table is a complex operation, involving many different steps, this // has been broken out into a utility to make it more obvious/easier to manipulate const tableSaveFunctions = new TableSaveFunctions({ - user: opts?.user, + userId: opts?.userId, oldTable, importRows: opts?.rowsToImport, }) From 15d124bfafbc8ad5de5d31f1bc73721d3d8fe0e8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 12:58:10 +0200 Subject: [PATCH 14/20] Move internal creation to sdk --- .../server/src/api/controllers/table/index.ts | 10 +++-- packages/server/src/sdk/app/tables/create.ts | 18 +++++++++ packages/server/src/sdk/app/tables/index.ts | 2 + .../src/sdk/app/tables/internal/index.ts | 39 ++++++++++++++++++- 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 packages/server/src/sdk/app/tables/create.ts diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 7bec1581b4..a1969f44fc 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -106,14 +106,18 @@ export async function save(ctx: UserCtx) { const isImport = table.rows const renaming = ctx.request.body._rename + const isCreate = !table._id + checkDefaultFields(table) - const api = pickApi({ table }) - let savedTable = await api.save(ctx, renaming) - if (!table._id) { + let savedTable: Table + if (isCreate) { + savedTable = await sdk.tables.create(table) savedTable = await sdk.tables.enrichViewSchemas(savedTable) await events.table.created(savedTable) } else { + const api = pickApi({ table }) + savedTable = await api.save(ctx, renaming) await events.table.updated(savedTable) } if (renaming) { diff --git a/packages/server/src/sdk/app/tables/create.ts b/packages/server/src/sdk/app/tables/create.ts new file mode 100644 index 0000000000..585623d0a4 --- /dev/null +++ b/packages/server/src/sdk/app/tables/create.ts @@ -0,0 +1,18 @@ +import { Row, Table } from "@budibase/types" + +// import * as external from "./external" +import * as internal from "./internal" +import { isExternal } from "./utils" + +export async function create( + table: Omit, + rows?: Row[], + userId?: string +): Promise { + if (isExternal({ table })) { + // const datasourceId = table.sourceId! + throw "not implemented" + // return await external.create(table, rows, userId) + } + return await internal.create(table, rows, userId) +} diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index fcf7051e7c..326b2e1456 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -1,5 +1,6 @@ import { populateExternalTableSchemas } from "./validation" import * as getters from "./getters" +import * as create from "./create" import * as updates from "./update" import * as utils from "./utils" import { migrate } from "./migration" @@ -7,6 +8,7 @@ import * as sqs from "./internal/sqs" export default { populateExternalTableSchemas, + ...create, ...updates, ...getters, ...utils, diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index 7c72834ee0..b2081fee96 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -5,6 +5,7 @@ import { ViewStatisticsSchema, ViewV2, Row, + TableSourceType, } from "@budibase/types" import { hasTypeChanged, @@ -15,14 +16,48 @@ import { EventType, updateLinks } from "../../../../db/linkedRows" import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" -import { context } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" import * as viewsSdk from "../../views" -import { getRowParams } from "../../../../db/utils" +import { generateTableID, getRowParams } from "../../../../db/utils" import { quotas } from "@budibase/pro" +export async function create(table: Table, rows?: Row[], userId?: string) { + const tableId = generateTableID() + + let tableToSave: Table = { + _id: tableId, + ...table, + // Ensure these fields are populated, even if not sent in the request + type: table.type || "table", + sourceType: TableSourceType.INTERNAL, + } + + const isImport = !!rows + + if (!tableToSave.views) { + tableToSave.views = {} + } + + try { + const { table } = await save(tableToSave, { + userId, + rowsToImport: rows, + isImport, + }) + + return table + } catch (err: any) { + if (err instanceof Error) { + throw new HTTPError(err.message, 400) + } else { + throw new HTTPError(err.message || err, err.status || 500) + } + } +} + export async function save( table: Table, opts?: { From 4efe335b65f6354d1f2b6a2a62c4ad39a0babfe4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 13:20:02 +0200 Subject: [PATCH 15/20] Create external table in sdk --- packages/server/src/sdk/app/tables/create.ts | 11 +++--- .../src/sdk/app/tables/external/index.ts | 36 +++++++++++++++++-- .../src/sdk/app/tables/internal/index.ts | 6 +++- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/tables/create.ts b/packages/server/src/sdk/app/tables/create.ts index 585623d0a4..ed6d6baeb0 100644 --- a/packages/server/src/sdk/app/tables/create.ts +++ b/packages/server/src/sdk/app/tables/create.ts @@ -1,6 +1,6 @@ import { Row, Table } from "@budibase/types" -// import * as external from "./external" +import * as external from "./external" import * as internal from "./internal" import { isExternal } from "./utils" @@ -9,10 +9,11 @@ export async function create( rows?: Row[], userId?: string ): Promise
{ + let createdTable: Table if (isExternal({ table })) { - // const datasourceId = table.sourceId! - throw "not implemented" - // return await external.create(table, rows, userId) + createdTable = await external.create(table) + } else { + createdTable = await internal.create(table, rows, userId) } - return await internal.create(table, rows, userId) + return createdTable } diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index e374e70c87..941d193b94 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -8,8 +8,11 @@ import { ViewV2, AutoFieldSubType, } from "@budibase/types" -import { context } from "@budibase/backend-core" -import { buildExternalTableId } from "../../../../integrations/utils" +import { context, HTTPError } from "@budibase/backend-core" +import { + breakExternalTableId, + buildExternalTableId, +} from "../../../../integrations/utils" import { foreignKeyStructure, hasTypeChanged, @@ -86,6 +89,35 @@ function validate(table: Table, oldTable?: Table) { } } +function getDatasourceId(table: Table) { + if (!table) { + throw new Error("No table supplied") + } + if (table.sourceId) { + return table.sourceId + } + if (!table._id) { + throw new Error("No table ID supplied") + } + return breakExternalTableId(table._id).datasourceId +} + +export async function create(table: Omit) { + const datasourceId = getDatasourceId(table) + + const tableToCreate = { ...table, created: true } + try { + const result = await save(datasourceId!, tableToCreate) + return result.table + } catch (err: any) { + if (err instanceof Error) { + throw new HTTPError(err.message, 400) + } else { + throw new HTTPError(err?.message || err, err.status || 500) + } + } +} + export async function save( datasourceId: string, update: Table, diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index b2081fee96..fbcbed03dc 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -24,7 +24,11 @@ import * as viewsSdk from "../../views" import { generateTableID, getRowParams } from "../../../../db/utils" import { quotas } from "@budibase/pro" -export async function create(table: Table, rows?: Row[], userId?: string) { +export async function create( + table: Omit, + rows?: Row[], + userId?: string +) { const tableId = generateTableID() let tableToSave: Table = { From 80ae7cbe0ba1b4f892e5a7cc8521947a09e36d94 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 13:21:22 +0200 Subject: [PATCH 16/20] Rename --- packages/server/src/api/controllers/table/external.ts | 2 +- packages/server/src/api/controllers/table/index.ts | 2 +- packages/server/src/api/controllers/table/internal.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 5b15d3d9c7..6f09bf4a61 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -31,7 +31,7 @@ function getDatasourceId(table: Table) { return breakExternalTableId(table._id).datasourceId } -export async function save( +export async function updateTable( ctx: UserCtx, renaming?: RenameColumn ) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index a1969f44fc..0e44548d99 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -117,7 +117,7 @@ export async function save(ctx: UserCtx) { await events.table.created(savedTable) } else { const api = pickApi({ table }) - savedTable = await api.save(ctx, renaming) + savedTable = await api.updateTable(ctx, renaming) await events.table.updated(savedTable) } if (renaming) { diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 256c5b1f0a..983f578bbc 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -12,7 +12,7 @@ import { } from "@budibase/types" import sdk from "../../../sdk" -export async function save( +export async function updateTable( ctx: UserCtx, renaming?: RenameColumn ) { From f05bf25e21e0246b9baa726d2ae71e7b5a09fa15 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 13:31:11 +0200 Subject: [PATCH 17/20] Add failing test --- packages/server/src/api/routes/tests/row.spec.ts | 12 +++++++++++- packages/server/src/tests/utilities/api/row.ts | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 207420bf9f..0995bd3824 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1846,7 +1846,7 @@ describe.each([ }) describe("exportRows", () => { - beforeAll(async () => { + beforeEach(async () => { table = await config.api.table.save(defaultTable()) }) @@ -1883,6 +1883,16 @@ describe.each([ }) }) + it("should allow exporting without filtering", async () => { + const existing = await config.api.row.save(table._id!, {}) + const res = await config.api.row.exportRows(table._id!) + const results = JSON.parse(res) + expect(results.length).toEqual(1) + const row = results[0] + + expect(row._id).toEqual(existing._id) + }) + it("should allow exporting only certain columns", async () => { const existing = await config.api.row.save(table._id!, {}) const res = await config.api.row.exportRows(table._id!, { diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 6bec59fdf7..52317e142a 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -105,7 +105,7 @@ export class RowAPI extends TestAPI { exportRows = async ( tableId: string, - body: ExportRowsRequest, + body?: ExportRowsRequest, format: RowExportFormat = RowExportFormat.JSON, expectations?: Expectations ) => { From 865b7a97e0f6c7f96d39a84f22255dd8e565c6bf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 13:31:34 +0200 Subject: [PATCH 18/20] Fix --- packages/server/src/sdk/app/rows/search/internal/internal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/internal/internal.ts b/packages/server/src/sdk/app/rows/search/internal/internal.ts index 6617fc376c..0ca10e82d4 100644 --- a/packages/server/src/sdk/app/rows/search/internal/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal/internal.ts @@ -62,10 +62,10 @@ export async function exportRows( ).rows.map(row => row.doc!) result = await outputProcessing(table, response) - } else if (query) { + } else { let searchResponse = await sdk.rows.search({ tableId, - query, + query: query || {}, sort, sortOrder, }) From 3b01f404dd500033ca71a23dae04557ac87b2283 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 9 Oct 2024 13:49:24 +0200 Subject: [PATCH 19/20] Fix imports --- packages/server/src/api/controllers/table/index.ts | 6 +++--- packages/server/src/api/controllers/table/internal.ts | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 0e44548d99..404f82e57c 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -102,8 +102,8 @@ export async function find(ctx: UserCtx) { export async function save(ctx: UserCtx) { const appId = ctx.appId - const table = ctx.request.body - const isImport = table.rows + const { rows, ...table } = ctx.request.body + const isImport = rows const renaming = ctx.request.body._rename const isCreate = !table._id @@ -112,7 +112,7 @@ export async function save(ctx: UserCtx) { let savedTable: Table if (isCreate) { - savedTable = await sdk.tables.create(table) + savedTable = await sdk.tables.create(table, rows, ctx.user._id) savedTable = await sdk.tables.enrichViewSchemas(savedTable) await events.table.created(savedTable) } else { diff --git a/packages/server/src/api/controllers/table/internal.ts b/packages/server/src/api/controllers/table/internal.ts index 983f578bbc..40ce5e279d 100644 --- a/packages/server/src/api/controllers/table/internal.ts +++ b/packages/server/src/api/controllers/table/internal.ts @@ -25,8 +25,6 @@ export async function updateTable( sourceType: rest.sourceType || TableSourceType.INTERNAL, } - const isImport = !!rows - if (!tableToSave.views) { tableToSave.views = {} } @@ -37,7 +35,6 @@ export async function updateTable( rowsToImport: rows, tableId: ctx.request.body._id, renaming, - isImport, }) return table From 4414da710ca22cd212df5d0040c30013715b1766 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 9 Oct 2024 11:53:39 +0000 Subject: [PATCH 20/20] Bump version to 2.32.14 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index faed02052e..9b765f17ae 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.13", + "version": "2.32.14", "npmClient": "yarn", "packages": [ "packages/*",