diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index c21a9b3fe5..d251ff672d 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,7 +3,6 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isInternalTableID, isInvalidISODateString, isValidFilter, isValidISODateString, @@ -497,9 +496,8 @@ class InternalBuilder { filterKey: string, whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { - const { relationships, endpoint, tableAliases: aliases } = this.query - const tableName = endpoint.entityId - const fromAlias = aliases?.[tableName] || tableName + const { relationships, endpoint, tableAliases: aliases, table } = this.query + const fromAlias = aliases?.[table.name] || table.name const matches = (value: string) => filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { @@ -1455,14 +1453,14 @@ class InternalBuilder { } qualifiedKnex(opts?: { alias?: string | boolean }): Knex.QueryBuilder { - let alias = this.query.tableAliases?.[this.query.endpoint.entityId] + let alias = this.query.tableAliases?.[this.query.table.name] if (opts?.alias === false) { alias = undefined } else if (typeof opts?.alias === "string") { alias = opts.alias } return this.knex( - this.tableNameWithSchema(this.query.endpoint.entityId, { + this.tableNameWithSchema(this.query.table.name, { alias, schema: this.query.endpoint.schema, }) @@ -1558,11 +1556,10 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships } = this.query + let { endpoint, filters, paginate, relationships, table } = this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT - const tableName = endpoint.entityId // start building the query let query = this.qualifiedKnex() // handle pagination @@ -1610,9 +1607,7 @@ class InternalBuilder { // handle relationships with a CTE for all others if (relationships?.length && aggregations.length === 0) { - const mainTable = - this.query.tableAliases?.[this.query.endpoint.entityId] || - this.query.endpoint.entityId + const mainTable = this.query.tableAliases?.[table.name] || table.name const cte = this.addSorting( this.knex .with("paginated", query) @@ -1622,7 +1617,7 @@ class InternalBuilder { }) ) // add JSON aggregations attached to the CTE - return this.addJsonRelationships(cte, tableName, relationships) + return this.addJsonRelationships(cte, table.name, relationships) } return query diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index ee5dfbf453..5b76e155f9 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -25,7 +25,7 @@ function generateSchema( schema: CreateTableBuilder, table: Table, tables: Record, - oldTable: null | Table = null, + oldTable?: Table, renamed?: RenameColumn ) { let primaryKeys = table && table.primary ? table.primary : [] @@ -55,7 +55,7 @@ function generateSchema( ) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct - const oldColumn = oldTable ? oldTable.schema[key] : null + const oldColumn = oldTable?.schema[key] if ( (oldColumn && oldColumn.type) || columnTypeSet.includes(key) || @@ -199,7 +199,7 @@ function buildUpdateTable( knex: SchemaBuilder, table: Table, tables: Record, - oldTable: Table, + oldTable?: Table, renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { @@ -281,7 +281,7 @@ class SqlTableQueryBuilder { client, json.table, json.tables, - json.table, + json.meta?.oldTable, json.meta?.renamed ) diff --git a/packages/pro b/packages/pro index 06e4506cef..8c5125b4f6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 06e4506cef37110e509a0ef0b9811ac07f8b2844 +Subproject commit 8c5125b4f6e53d0a43d4de90aecbf53d9f7da8ff diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 422896cd52..5f01f323e6 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -137,7 +137,7 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { datasource: datasourceId, entityId: tableName, operation } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -706,8 +706,8 @@ export class ExternalRequest { let json: QueryJson = { endpoint: { - datasource: this.datasource, - entityId: table.name, + datasourceId: this.datasource, + entityId: table, operation, }, resource: { diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index ca4fcb1d16..df5707b072 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -11,19 +11,23 @@ export async function makeTableRequest( datasource: Datasource, operation: Operation, table: Table, + oldTable?: Table, renamed?: RenameColumn ) { const json: QueryJson = { endpoint: { - datasource, - entityId: table._id!, + datasourceId: datasource, + entityId: table, operation, }, } + if (!json.meta) { + json.meta = {} + } + if (oldTable) { + json.meta.oldTable = oldTable + } if (renamed) { - if (!json.meta) { - json.meta = {} - } json.meta.renamed = renamed } return makeExternalQuery(json) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 6f09bf4a61..5e77dc1dc6 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -54,7 +54,7 @@ export async function updateTable( return table } catch (err: any) { if (err instanceof Error) { - ctx.throw(400, err.message) + throw err } else { ctx.throw(err.status || 500, err?.message || err) } diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 9130034501..44b21e0350 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -837,7 +837,7 @@ if (descriptions.length) { const res = await config.api.datasource.query({ endpoint: { - datasource: datasource._id!, + datasourceId: datasource._id!, operation: Operation.READ, entityId, }, diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d8bf0e376d..6492a651e1 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -382,7 +382,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { } async query(json: EnrichedQueryJson): Promise { - const sheet = json.endpoint.entityId + const sheet = json.table.name switch (json.endpoint.operation) { case Operation.CREATE: return this.create({ sheet, row: json.body as Row }) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 4343593d1e..d690a0e17e 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -580,7 +580,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { operation !== Operation.DELETE ) { const lastRow = await this.internalQuery({ - sql: `SELECT * FROM "${json.endpoint.entityId}" WHERE ROWID = '${response.lastRowid}'`, + sql: `SELECT * FROM "${json.table.name}" WHERE ROWID = '${response.lastRowid}'`, }) return lastRow.rows as Row[] } else { diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index c06ba2d35c..4de84f689e 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -231,8 +231,7 @@ describe("Captures of real examples", () => { }, queryJson) expect(returningQuery).toEqual({ sql: multiline( - `select top (@p0) * from [people] as [a] where CASE WHEN [a].[name] = @p1 THEN 1 ELSE 0 END = 1 and - CASE WHEN [a].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [a].[name] asc` + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` ), bindings: [1, "Test", 22], }) @@ -270,7 +269,7 @@ describe("Captures of real examples", () => { fields: string[] = ["a"] ): EnrichedQueryJson { return { - endpoint: { datasource: "", entityId: "", operation: op }, + endpoint: { datasourceId: "", entityId: "", operation: op }, resource: { fields, }, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index d6ac2a5862..ba5614edca 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -358,7 +358,7 @@ export async function search( const request: QueryJson = { endpoint: { // not important, we query ourselves - datasource: SQS_DATASOURCE_INTERNAL, + datasourceId: SQS_DATASOURCE_INTERNAL, entityId: table._id!, operation: Operation.READ, }, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 2f9bea3c91..62e0db9aad 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -222,7 +222,7 @@ export default class AliasTables { aliases: this.aliasMap([ relationship.through, relationship.tableName, - json.endpoint.entityId, + json.table.name, ]), })) } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 17bbebb251..977cdb9bd2 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -23,8 +23,6 @@ import { isSQL } from "../../../integrations/utils" import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" -import { breakExternalTableId } from "@budibase/backend-core/src/sql/utils" -import { isDatasourceId } from "@budibase/backend-core/src/docIds" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -90,18 +88,15 @@ export async function enrichQueryJson( json: QueryJson ): Promise { let datasource: Datasource | undefined = undefined - let entityId = json.endpoint.entityId - if (typeof json.endpoint.datasource === "string") { - if (json.endpoint.datasource !== SQS_DATASOURCE_INTERNAL) { - datasource = await sdk.datasources.get(json.endpoint.datasource, { + + if (typeof json.endpoint.datasourceId === "string") { + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId, { enriched: true, }) } } else { - datasource = json.endpoint.datasource - if (isDatasourceId(entityId)) { - entityId = breakExternalTableId(entityId).tableName - } + datasource = json.endpoint.datasourceId } let tables: Record @@ -111,7 +106,16 @@ export async function enrichQueryJson( tables = processInternalTables(await sdk.tables.getAllInternalTables()) } - const table = tables[entityId] + let table: Table + if (typeof json.endpoint.entityId === "string") { + let entityId = json.endpoint.entityId + if (docIds.isDatasourceId(entityId)) { + entityId = sql.utils.breakExternalTableId(entityId).tableName + } + table = tables[entityId] + } else { + table = json.endpoint.entityId + } return { table, diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 5d8f03a7a6..3511222678 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -241,7 +241,13 @@ export async function save( } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE - await makeTableRequest(datasource, operation, tableToSave, opts?.renaming) + await makeTableRequest( + datasource, + operation, + tableToSave, + oldTable, + opts?.renaming + ) // update any extra tables (like foreign keys in other tables) for (let extraTable of extraTablesToUpdate) { const oldExtraTable = oldTables[extraTable.name] diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 72f7de455a..e25390bdd0 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -158,8 +158,8 @@ export interface ManyToManyRelationshipJson { export interface QueryJson { endpoint: { - datasource: string | Datasource - entityId: string + datasourceId: string | Datasource + entityId: string | Table operation: Operation schema?: string } @@ -173,6 +173,7 @@ export interface QueryJson { body?: Row | Row[] meta?: { renamed?: RenameColumn + oldTable?: Table // can specify something that columns could be prefixed with columnPrefix?: string } @@ -190,7 +191,7 @@ export interface EnrichedQueryJson extends QueryJson { } export interface QueryJsonRequest extends Omit { - endpoint: QueryJson["endpoint"] & { datasource: string } + endpoint: QueryJson["endpoint"] & { datasourceId: string; entityId: string } } export interface QueryOptions {