diff --git a/lerna.json b/lerna.json index c06173fe04..6fb032ac77 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.0", + "version": "2.21.2", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index 01473ad991..213c65e18e 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -184,7 +184,7 @@ export async function getRole( return cloneDeep(BUILTIN_ROLES.PUBLIC) } // only throw an error if there is no role at all - if (!role || Object.keys(role).length === 0) { + if (Object.keys(role || {}).length === 0) { throw err } } diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 7cd571f6d9..1fbd0df522 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -1,7 +1,7 @@ ) { diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts index 9658a0d638..46b090bb97 100644 --- a/packages/server/src/api/controllers/row/alias.ts +++ b/packages/server/src/api/controllers/row/alias.ts @@ -62,7 +62,11 @@ export default class AliasTables { if (idx === -1 || idx > 1) { return } - return Math.abs(tableName.length - name.length) <= 2 + // this might look a bit mad, but the idea is if the field is wrapped, say in "", `` or [] + // then the idx of the table name will be 1, and we should allow for it ending in a closing + // character - otherwise it should be the full length if the index is zero + const allowedCharacterDiff = idx * 2 + return Math.abs(tableName.length - name.length) <= allowedCharacterDiff }) if (foundTableName) { const aliasedTableName = tableName.replace( @@ -107,57 +111,55 @@ export default class AliasTables { } async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse { - json = cloneDeep(json) - const aliasTable = (table: Table) => ({ - ...table, - name: this.getAlias(table.name), - }) - // run through the query json to update anywhere a table may be used - if (json.resource?.fields) { - json.resource.fields = json.resource.fields.map(field => - this.aliasField(field) - ) - } - if (json.filters) { - for (let [filterKey, filter] of Object.entries(json.filters)) { - if (typeof filter !== "object") { - continue - } - const aliasedFilters: typeof filter = {} - for (let key of Object.keys(filter)) { - aliasedFilters[this.aliasField(key)] = filter[key] - } - json.filters[filterKey as keyof SearchFilters] = aliasedFilters + const fieldLength = json.resource?.fields?.length + const aliasingEnabled = fieldLength && fieldLength > 0 + if (aliasingEnabled) { + json = cloneDeep(json) + // run through the query json to update anywhere a table may be used + if (json.resource?.fields) { + json.resource.fields = json.resource.fields.map(field => + this.aliasField(field) + ) } - } - if (json.relationships) { - json.relationships = json.relationships.map(relationship => ({ - ...relationship, - aliases: this.aliasMap([ - relationship.through, - relationship.tableName, - json.endpoint.entityId, - ]), - })) - } - if (json.meta?.table) { - json.meta.table = aliasTable(json.meta.table) - } - if (json.meta?.tables) { - const aliasedTables: Record = {} - for (let [tableName, table] of Object.entries(json.meta.tables)) { - aliasedTables[this.getAlias(tableName)] = aliasTable(table) + if (json.filters) { + for (let [filterKey, filter] of Object.entries(json.filters)) { + if (typeof filter !== "object") { + continue + } + const aliasedFilters: typeof filter = {} + for (let key of Object.keys(filter)) { + aliasedFilters[this.aliasField(key)] = filter[key] + } + json.filters[filterKey as keyof SearchFilters] = aliasedFilters + } } - json.meta.tables = aliasedTables + if (json.meta?.table) { + this.getAlias(json.meta.table.name) + } + if (json.meta?.tables) { + Object.keys(json.meta.tables).forEach(tableName => + this.getAlias(tableName) + ) + } + if (json.relationships) { + json.relationships = json.relationships.map(relationship => ({ + ...relationship, + aliases: this.aliasMap([ + relationship.through, + relationship.tableName, + json.endpoint.entityId, + ]), + })) + } + // invert and return + const invertedTableAliases: Record = {} + for (let [key, value] of Object.entries(this.tableAliases)) { + invertedTableAliases[value] = key + } + json.tableAliases = invertedTableAliases } - // invert and return - const invertedTableAliases: Record = {} - for (let [key, value] of Object.entries(this.tableAliases)) { - invertedTableAliases[value] = key - } - json.tableAliases = invertedTableAliases const response = await getDatasourceAndQuery(json) - if (Array.isArray(response)) { + if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { return response diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6605052598..c8acb606b3 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -12,6 +12,8 @@ import { } from "@budibase/types" import environment from "../../environment" +type QueryFunction = (query: Knex.SqlNative, operation: Operation) => any + const envLimit = environment.SQL_MAX_ROWS ? parseInt(environment.SQL_MAX_ROWS) : null @@ -322,15 +324,18 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json const table = json.meta?.table + const aliases = json.tableAliases + const aliased = + table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name if (sort && Object.keys(sort || {}).length > 0) { for (let [key, value] of Object.entries(sort)) { const direction = value.direction === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(`${table?.name}.${key}`, direction) + query = query.orderBy(`${aliased}.${key}`, direction) } } else if (this.client === SqlClient.MS_SQL && paginate?.limit) { // @ts-ignore - query = query.orderBy(`${table?.name}.${table?.primary[0]}`) + query = query.orderBy(`${aliased}.${table?.primary[0]}`) } return query } @@ -605,7 +610,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return query.toSQL().toNative() } - async getReturningRow(queryFn: Function, json: QueryJson) { + async getReturningRow(queryFn: QueryFunction, json: QueryJson) { if (!json.extra || !json.extra.idFilter) { return {} } @@ -617,7 +622,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { resource: { fields: [], }, - filters: json.extra.idFilter, + filters: json.extra?.idFilter, paginate: { limit: 1, }, @@ -646,7 +651,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // this function recreates the returning functionality of postgres async queryWithReturning( json: QueryJson, - queryFn: Function, + queryFn: QueryFunction, processFn: Function = (result: any) => result ) { const sqlClient = this.getSqlClient() diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9b3f6a1b38..fe9798aaa0 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -4,6 +4,7 @@ import Sql from "../base/sql" import { SqlClient } from "../utils" import AliasTables from "../../api/controllers/row/alias" import { generator } from "@budibase/backend-core/tests" +import { Knex } from "knex" function multiline(sql: string) { return sql.replace(/\n/g, "").replace(/ +/g, " ") @@ -160,6 +161,28 @@ describe("Captures of real examples", () => { }) }) + describe("returning (everything bar Postgres)", () => { + it("should be able to handle row returning", () => { + const queryJson = getJson("createSimple.json") + const SQL = new Sql(SqlClient.MS_SQL, limit) + let query = SQL._query(queryJson, { disableReturning: true }) + expect(query).toEqual({ + sql: "insert into [people] ([age], [name]) values (@p0, @p1)", + bindings: [22, "Test"], + }) + + // now check returning + let returningQuery: Knex.SqlNative = { sql: "", bindings: [] } + SQL.getReturningRow((input: Knex.SqlNative) => { + returningQuery = input + }, queryJson) + expect(returningQuery).toEqual({ + sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]", + bindings: [1, "Test", 22], + }) + }) + }) + describe("check max character aliasing", () => { it("should handle over 'z' max character alias", () => { const tableNames = [] diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index 3445f5fe67..ba7fa4ef9b 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -68,7 +68,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json new file mode 100644 index 0000000000..33a88d30e1 --- /dev/null +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -0,0 +1,64 @@ +{ + "endpoint": { + "datasourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "entityId": "people", + "operation": "CREATE" + }, + "resource": { + "fields": [ + "a.name", + "a.age" + ] + }, + "filters": {}, + "relationships": [], + "body": { + "name": "Test", + "age": 22 + }, + "extra": { + "idFilter": { + "equal": { + "name": "Test", + "age": 22 + } + } + }, + "meta": { + "table": { + "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", + "type": "table", + "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "sourceType": "external", + "primary": [ + "name", + "age" + ], + "name": "people", + "schema": { + "name": { + "type": "string", + "externalType": "varchar", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": true + } + }, + "age": { + "type": "number", + "externalType": "int", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + } + }, + "primaryDisplay": "name" + } + }, + "tableAliases": { + "people": "a" + } +} \ No newline at end of file diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 20331b949a..82d85c417b 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -58,7 +58,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index 2266b8c8be..d6e099c4b6 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -34,7 +34,7 @@ "keypartone", "keyparttwo" ], - "name": "a", + "name": "compositetable", "schema": { "keyparttwo": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index ee658aed18..d71f0552c6 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -49,7 +49,7 @@ "primary": [ "taskid" ], - "name": "a", + "name": "tasks", "schema": { "executorid": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index 682ebaab2d..cec2fdb025 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -63,7 +63,7 @@ "primary": [ "productid" ], - "name": "a", + "name": "products", "schema": { "productname": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index eb1025f382..399cb0f4d2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -53,7 +53,7 @@ "primary": [ "productid" ], - "name": "a", + "name": "products", "schema": { "productname": { "type": "string", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index afa0889450..2b5d156546 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -109,7 +109,7 @@ "primary": [ "taskid" ], - "name": "a", + "name": "tasks", "schema": { "executorid": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 01e795bd6c..42c2a44335 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -66,7 +66,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 01e795bd6c..42c2a44335 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -66,7 +66,7 @@ "primary": [ "personid" ], - "name": "a", + "name": "persons", "schema": { "year": { "type": "number", diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e2d1a1b32c..e95b904767 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -11,7 +11,10 @@ import { import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" import { handleRequest } from "../../../../api/controllers/row/external" -import { breakExternalTableId } from "../../../../integrations/utils" +import { + breakExternalTableId, + breakRowIdField, +} from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "../search" @@ -52,6 +55,15 @@ export async function search(options: SearchParams) { } } + // Make sure oneOf _id queries decode the Row IDs + if (query?.oneOf?._id) { + const rowIds = query.oneOf._id + query.oneOf._id = rowIds.map((row: string) => { + const ids = breakRowIdField(row) + return ids[0] + }) + } + try { const table = await sdk.tables.getTable(tableId) options = searchInputMapping(table, options) @@ -119,9 +131,7 @@ export async function exportRows( requestQuery = { oneOf: { _id: rowIds.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") - ) + const ids = breakRowIdField(row) if (ids.length > 1) { throw new HTTPError( "Export data does not support composite keys.", diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index 1aaea8e258..bae84592ca 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -21,10 +21,11 @@ jest.unmock("mysql2/promise") jest.setTimeout(30000) -describe.skip("external", () => { +describe("external search", () => { const config = new TestConfiguration() let externalDatasource: Datasource, tableData: Table + const rows: Row[] = [] beforeAll(async () => { const container = await new GenericContainer("mysql") @@ -89,67 +90,81 @@ describe.skip("external", () => { }, }, } + + const table = await config.createExternalTable({ + ...tableData, + sourceId: externalDatasource._id, + }) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + tableId: table._id, + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } }) - describe("search", () => { - const rows: Row[] = [] - beforeAll(async () => { - const table = await config.createExternalTable({ - ...tableData, - sourceId: externalDatasource._id, - }) - for (let i = 0; i < 10; i++) { - rows.push( - await config.createRow({ - tableId: table._id, - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) }) + }) - it("default search returns all the data", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! - const searchParams: SearchParams = { - tableId, - query: {}, - } - const result = await search(searchParams) + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyExternalColsAttributes, + name: r.name, + age: r.age, + })) ) - }) + ) }) + }) - it("querying by fields will always return data attribute columns", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! + it("will decode _id in oneOf query", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! - const searchParams: SearchParams = { - tableId, - query: {}, - fields: ["name", "age"], - } - const result = await search(searchParams) + const searchParams: SearchParams = { + tableId, + query: { + oneOf: { + _id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"], + }, + }, + } + const result = await search(searchParams) - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining( - rows.map(r => ({ - ...expectAnyExternalColsAttributes, - name: r.name, - age: r.age, - })) - ) - ) - }) + expect(result.rows).toHaveLength(3) + expect(result.rows.map(row => row.id)).toEqual([1, 4, 8]) }) }) }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 4eee3cea41..5d93dcaca2 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -1,6 +1,5 @@ import { FieldType, - FieldTypeSubtypes, SearchParams, Table, DocumentType,