From 594af57339a9cac886f766abc694475f18fc45e5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 26 Nov 2024 16:28:22 +0000 Subject: [PATCH] wip --- packages/backend-core/src/sql/sql.ts | 60 +++-- packages/backend-core/src/sql/sqlTable.ts | 24 +- .../api/controllers/row/ExternalRequest.ts | 56 ++--- .../server/src/integrations/base/query.ts | 40 ++-- .../src/integrations/microsoftSqlServer.ts | 10 +- packages/server/src/integrations/mysql.ts | 8 +- packages/server/src/integrations/oracle.ts | 4 +- packages/server/src/integrations/postgres.ts | 4 +- .../basicFetchWithRelationships.json | 208 +++++++++-------- .../tests/sqlQueryJson/createSimple.json | 64 +++--- .../sqlQueryJson/createWithRelationships.json | 215 +++++++++-------- .../tests/sqlQueryJson/deleteSimple.json | 83 +++---- .../sqlQueryJson/enrichRelationship.json | 143 ++++++------ .../tests/sqlQueryJson/fetchManyToMany.json | 78 +++---- .../sqlQueryJson/filterByRelationship.json | 78 +++---- .../sqlQueryJson/manyRelationshipFilters.json | 168 +++++++------- .../sqlQueryJson/updateRelationship.json | 217 +++++++++--------- .../tests/sqlQueryJson/updateSimple.json | 217 +++++++++--------- .../src/sdk/app/rows/search/internal/sqs.ts | 23 +- packages/server/src/sdk/app/rows/sqlAlias.ts | 28 ++- packages/server/src/sdk/app/rows/utils.ts | 36 ++- packages/types/src/sdk/datasources.ts | 4 +- packages/types/src/sdk/search.ts | 13 +- 23 files changed, 854 insertions(+), 927 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e97e31869c..c897c106b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -18,6 +18,7 @@ import { BasicOperator, BBReferenceFieldMetadata, CalculationType, + EnrichedQueryJson, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -27,7 +28,6 @@ import { LogicalOperator, Operation, prefixed, - QueryJson, QueryOptions, RangeOperator, RelationshipsJson, @@ -134,18 +134,18 @@ const allowEmptyRelationships: Record = { class InternalBuilder { private readonly client: SqlClient - private readonly query: QueryJson + private readonly query: EnrichedQueryJson private readonly splitter: dataFilters.ColumnSplitter private readonly knex: Knex - constructor(client: SqlClient, knex: Knex, query: QueryJson) { + constructor(client: SqlClient, knex: Knex, query: EnrichedQueryJson) { this.client = client this.query = query this.knex = knex this.splitter = new dataFilters.ColumnSplitter([this.table], { aliases: this.query.tableAliases, - columnPrefix: this.query.meta.columnPrefix, + columnPrefix: this.query.meta?.columnPrefix, }) } @@ -167,7 +167,7 @@ class InternalBuilder { } get table(): Table { - return this.query.meta.table + return this.query.table } get knexClient(): Knex.Client { @@ -273,8 +273,7 @@ class InternalBuilder { } private isFullSelectStatementRequired(): boolean { - const { meta } = this.query - for (let column of Object.values(meta.table.schema)) { + for (let column of Object.values(this.table.schema)) { if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { return true } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { @@ -285,14 +284,14 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource } = this.query + const { endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } const alias = this.getTableName(endpoint.entityId) - const schema = meta.table.schema + const schema = this.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw("??", [`${alias}.*`])] } @@ -1018,15 +1017,15 @@ class InternalBuilder { const name = tableOrName if (this.query.table?.name === name) { table = this.query.table - } else if (this.query.meta.table?.name === name) { - table = this.query.meta.table - } else if (!this.query.meta.tables?.[name]) { + } else if (this.query.table.name === name) { + table = this.query.table + } else if (!this.query.tables[name]) { // This can legitimately happen in custom queries, where the user is // querying against a table that may not have been imported into // Budibase. return name } else { - table = this.query.meta.tables[name] + table = this.query.tables[name] } } else if (tableOrName) { table = tableOrName @@ -1238,7 +1237,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint, meta } = this.query + const { resource, tableAliases: aliases, endpoint, tables } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -1253,7 +1252,7 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } - const relatedTable = meta.tables?.[toTable] + const relatedTable = tables[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable, throughAlias = (throughTable && aliases?.[throughTable]) || throughTable @@ -1276,7 +1275,7 @@ class InternalBuilder { Math.floor(this.maxFunctionParameters() / 2) ) const fieldList = relationshipFields.map(field => - this.buildJsonField(relatedTable!, field) + this.buildJsonField(relatedTable, field) ) const fieldListFormatted = fieldList .map(f => { @@ -1472,9 +1471,7 @@ class InternalBuilder { if (this.client === SqlClient.ORACLE) { // Oracle doesn't seem to automatically insert nulls // if we don't specify them, so we need to do that here - for (const [column, schema] of Object.entries( - this.query.meta.table.schema - )) { + for (const [column, schema] of Object.entries(this.query.table.schema)) { if ( schema.constraints?.presence === true || schema.type === FieldType.FORMULA || @@ -1678,7 +1675,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes. * @return the query ready to be passed to the driver. */ - _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { + _query( + json: EnrichedQueryJson, + opts: QueryOptions = {} + ): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { client: this.getBaseSqlClient(), @@ -1728,34 +1728,30 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return this.convertToNative(query, opts) } - async getReturningRow(queryFn: QueryFunction, json: QueryJson) { + async getReturningRow(queryFn: QueryFunction, json: EnrichedQueryJson) { if (!json.extra || !json.extra.idFilter) { return {} } const input = this._query({ + ...json, endpoint: { ...json.endpoint, operation: Operation.READ, }, - resource: { - fields: [], - }, + resource: { fields: [] }, filters: json.extra?.idFilter, - paginate: { - limit: 1, - }, - meta: json.meta, + paginate: { limit: 1 }, }) return queryFn(input, Operation.READ) } // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row - checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta.table || !json.meta.table.primary) { + checkLookupKeys(id: any, json: EnrichedQueryJson) { + if (!id || !json.table.primary) { return json } - const primaryKey = json.meta.table.primary?.[0] + const primaryKey = json.table.primary[0] json.extra = { idFilter: { equal: { @@ -1768,7 +1764,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // this function recreates the returning functionality of postgres async queryWithReturning( - json: QueryJson, + json: EnrichedQueryJson, queryFn: QueryFunction, processFn: Function = (result: any) => result ) { diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 84f4e290aa..ee5dfbf453 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -3,13 +3,13 @@ import { FieldType, NumberFieldMetadata, Operation, - QueryJson, RelationshipType, RenameColumn, SqlQuery, Table, TableSourceType, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { breakExternalTableId, getNativeSql } from "./utils" import { helpers, utils } from "@budibase/shared-core" @@ -200,7 +200,7 @@ function buildUpdateTable( table: Table, tables: Record, oldTable: Table, - renamed: RenameColumn + renamed?: RenameColumn ): SchemaBuilder { return knex.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable, renamed) @@ -238,11 +238,11 @@ class SqlTableQueryBuilder { * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. */ - _operation(json: QueryJson): Operation { + _operation(json: EnrichedQueryJson): Operation { return json.endpoint.operation } - _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] { + _tableQuery(json: EnrichedQueryJson): SqlQuery | SqlQuery[] { let client = knex({ client: this.sqlClient }).schema let schemaName = json?.endpoint?.schema if (schemaName) { @@ -250,7 +250,7 @@ class SqlTableQueryBuilder { } let query: Knex.SchemaBuilder - if (!json.table || !json.meta || !json.meta.tables) { + if (!json.table || !json.tables) { throw new Error("Cannot execute without table being specified") } if (json.table.sourceType === TableSourceType.INTERNAL) { @@ -259,14 +259,14 @@ class SqlTableQueryBuilder { switch (this._operation(json)) { case Operation.CREATE_TABLE: - query = buildCreateTable(client, json.table, json.meta.tables) + query = buildCreateTable(client, json.table, json.tables) break case Operation.UPDATE_TABLE: - if (!json.meta || !json.meta.table) { + if (!json.table) { throw new Error("Must specify old table for update") } // renameColumn does not work for MySQL, so return a raw query - if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MY_SQL && json.meta?.renamed) { const updatedColumn = json.meta.renamed.updated const tableName = schemaName ? `\`${schemaName}\`.\`${json.table.name}\`` @@ -280,14 +280,14 @@ class SqlTableQueryBuilder { query = buildUpdateTable( client, json.table, - json.meta.tables, - json.meta.table, - json.meta.renamed! + json.tables, + json.table, + json.meta?.renamed ) // renameColumn for SQL Server returns a parameterised `sp_rename` query, // which is not supported by SQL Server and gives a syntax error. - if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) { + if (this.sqlClient === SqlClient.MS_SQL && json.meta?.renamed) { const oldColumn = json.meta.renamed.old const updatedColumn = json.meta.renamed.updated const tableName = schemaName diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 939ea399d9..baf1665cbe 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -42,7 +42,7 @@ import { sqlOutputProcessing, } from "./utils" import { - getDatasourceAndQuery, + enrichQueryJson, processRowCountResponse, } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" @@ -135,16 +135,9 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig { return config } -function getEndpoint(tableId: string | undefined, operation: string) { - if (!tableId) { - throw new Error("Cannot get endpoint information - no table ID specified") - } +function getEndpoint(tableId: string, operation: Operation) { const { datasourceId, tableName } = breakExternalTableId(tableId) - return { - datasourceId: datasourceId, - entityId: tableName, - operation: operation as Operation, - } + return { datasourceId, entityId: tableName, operation } } function isOneSide( @@ -268,12 +261,9 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.DELETE), filters, - meta: { - table, - }, }) } else { return [] @@ -289,13 +279,10 @@ export class ExternalRequest { const filters = this.prepareFilters(rowId, {}, table) // safety check, if there are no filters on deletion bad things happen if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ + return makeExternalQuery({ endpoint: getEndpoint(tableId, Operation.UPDATE), body: { [colName]: null }, filters, - meta: { - table, - }, }) } else { return [] @@ -311,12 +298,9 @@ export class ExternalRequest { } async getRow(table: Table, rowId: string): Promise { - const response = await getDatasourceAndQuery({ + const response = await makeExternalQuery({ endpoint: getEndpoint(table._id!, Operation.READ), filters: this.prepareFilters(rowId, {}, table), - meta: { - table, - }, }) if (Array.isArray(response) && response.length > 0) { return response[0] @@ -490,16 +474,13 @@ export class ExternalRequest { if (!relatedTable) { throw new Error("unable to find related table") } - const response = await getDatasourceAndQuery({ - endpoint: endpoint, + const response = await makeExternalQuery({ + endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, - meta: { - table: relatedTable, - }, }) // this is the response from knex if no rows found const rows: Row[] = @@ -537,6 +518,11 @@ export class ExternalRequest { for (let relationship of relationships) { const { key, tableId, isUpdate, id, relationshipType, ...rest } = relationship + + if (!tableId) { + throw new Error("Table ID is unknown, cannot find table") + } + const body: { [key: string]: any } = processObjectSync(rest, row, {}) const linkTable = this.getTable(tableId) const relationshipPrimary = linkTable?.primary || [] @@ -583,14 +569,11 @@ export class ExternalRequest { const operation = isUpdate ? Operation.UPDATE : Operation.CREATE if (!existingRelationship) { promises.push( - getDatasourceAndQuery({ + makeExternalQuery({ endpoint: getEndpoint(tableId, operation), // if we're doing many relationships then we're writing, only one response body, filters: this.prepareFilters(id, {}, linkTable), - meta: { - table: linkTable, - }, }) ) } else { @@ -749,10 +732,6 @@ export class ExternalRequest { table ), }, - meta: { - table, - tables: this.tables, - }, } // remove any relationships that could block deletion @@ -773,8 +752,11 @@ export class ExternalRequest { response = [unprocessedRow] } else { response = env.SQL_ALIASING_DISABLE - ? await getDatasourceAndQuery(json) - : await aliasing.queryWithAliasing(json, makeExternalQuery) + ? await makeExternalQuery(json) + : await aliasing.queryWithAliasing( + await enrichQueryJson(json), + makeExternalQuery + ) } // if it's a counting operation there will be no more processing, just return the number diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 55886cd20f..2185eace2c 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -1,37 +1,27 @@ -import { - QueryJson, - Datasource, - DatasourcePlusQueryResponse, - RowOperations, -} from "@budibase/types" +import { DatasourcePlusQueryResponse, QueryJson } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" +import { enrichQueryJson } from "../../sdk/app/rows/utils" export async function makeExternalQuery( - datasource: Datasource, json: QueryJson ): Promise { - const entityId = json.endpoint.entityId, - tableName = json.meta.table.name, - tableId = json.meta.table._id - // case found during testing - make sure this doesn't happen again - if ( - RowOperations.includes(json.endpoint.operation) && - entityId !== tableId && - entityId !== tableName - ) { - throw new Error("Entity ID and table metadata do not align") - } - if (!datasource) { + const enrichedJson = await enrichQueryJson(json) + if (!enrichedJson.datasource) { throw new Error("No datasource provided for external query") } - datasource = await sdk.datasources.enrich(datasource) - const Integration = await getIntegration(datasource.source) + + enrichedJson.datasource = await sdk.datasources.enrich( + enrichedJson.datasource + ) + + const Integration = await getIntegration(enrichedJson.datasource.source) + // query is the opinionated function - if (Integration.prototype.query) { - const integration = new Integration(datasource.config) - return integration.query(json) - } else { + if (!Integration.prototype.query) { throw "Datasource does not support query." } + + const integration = new Integration(enrichedJson.datasource.config) + return integration.query(json) } diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 4997f3256d..a7a0b53312 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -4,9 +4,9 @@ import { DatasourceFieldType, DatasourcePlus, DatasourcePlusQueryResponse, + EnrichedQueryJson, Integration, Operation, - QueryJson, QueryType, Schema, SourceName, @@ -506,7 +506,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { return response.recordset || [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const schema = this.config.schema await this.connect() if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) { @@ -515,14 +515,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { const operation = this._operation(json) const queryFn = (query: any, op: string) => this.internalQuery(query, op) const processFn = (result: any) => { - if (json?.meta?.table && result.recordset) { + if (result.recordset) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result.recordset, json.tableAliases ) - } else if (result.recordset) { - return result.recordset } return [{ [operation]: true }] } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f78915fb49..9236409375 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -2,7 +2,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, TableSchema, @@ -15,6 +14,7 @@ import { DatasourcePlusQueryResponse, SqlQueryBinding, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -390,15 +390,15 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return results.length ? results : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { await this.connect() try { const queryFn = (query: any) => this.internalQuery(query, { connect: false, disableCoercion: true }) const processFn = (result: any) => { - if (json?.meta?.table && Array.isArray(result)) { + if (Array.isArray(result)) { return this.convertJsonStringColumns( - json.meta.table, + json.table, result, json.tableAliases ) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index fcbc0731ea..b48a95e3e6 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,7 +3,6 @@ import { DatasourceFieldType, Integration, Operation, - QueryJson, QueryType, SqlQuery, Table, @@ -15,6 +14,7 @@ import { Row, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { buildExternalTableId, @@ -545,7 +545,7 @@ class OracleIntegration extends Sql implements DatasourcePlus { : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json) const input = this._query(json, { disableReturning: true }) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 28400f616f..164468b5ae 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -3,7 +3,6 @@ import { Integration, DatasourceFieldType, QueryType, - QueryJson, SqlQuery, Table, DatasourcePlus, @@ -14,6 +13,7 @@ import { TableSourceType, DatasourcePlusQueryResponse, SqlClient, + EnrichedQueryJson, } from "@budibase/types" import { getSqlQuery, @@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus { return response.rows.length ? response.rows : [{ deleted: true }] } - async query(json: QueryJson): Promise { + async query(json: EnrichedQueryJson): Promise { const operation = this._operation(json).toLowerCase() const input = this._query(json) as SqlQuery if (Array.isArray(input)) { diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json index aa2cab3afd..27d6efec29 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json @@ -61,113 +61,111 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": ["personid"], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": ["support", "designer", "programmer", "qa"] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json index 33a88d30e1..474be32cf2 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createSimple.json @@ -5,10 +5,7 @@ "operation": "CREATE" }, "resource": { - "fields": [ - "a.name", - "a.age" - ] + "fields": ["a.name", "a.age"] }, "filters": {}, "relationships": [], @@ -24,41 +21,36 @@ } } }, - "meta": { - "table": { - "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", - "type": "table", - "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", - "sourceType": "external", - "primary": [ - "name", - "age" - ], - "name": "people", - "schema": { - "name": { - "type": "string", - "externalType": "varchar", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": true - } - }, - "age": { - "type": "number", - "externalType": "int", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } + "table": { + "_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people", + "type": "table", + "sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae", + "sourceType": "external", + "primary": ["name", "age"], + "name": "people", + "schema": { + "name": { + "type": "string", + "externalType": "varchar", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": true } }, - "primaryDisplay": "name" - } + "age": { + "type": "number", + "externalType": "int", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + } + }, + "primaryDisplay": "name" }, "tableAliases": { "people": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json index 82d85c417b..fbe0b468e9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json @@ -52,122 +52,113 @@ "idFilter": {} }, "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json index d6e099c4b6..d9fc34165f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json @@ -5,11 +5,7 @@ "operation": "DELETE" }, "resource": { - "fields": [ - "a.keyparttwo", - "a.keypartone", - "a.name" - ] + "fields": ["a.keyparttwo", "a.keypartone", "a.name"] }, "filters": { "equal": { @@ -26,50 +22,45 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", - "primary": [ - "keypartone", - "keyparttwo" - ], - "name": "compositetable", - "schema": { - "keyparttwo": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keyparttwo", - "constraints": { - "presence": true - } - }, - "keypartone": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "keypartone", - "constraints": { - "presence": true - } - }, - "name": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "name", - "constraints": { - "presence": false - } + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable", + "primary": ["keypartone", "keyparttwo"], + "name": "compositetable", + "schema": { + "keyparttwo": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keyparttwo", + "constraints": { + "presence": true } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "keypartone" - } + "keypartone": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "keypartone", + "constraints": { + "presence": true + } + }, + "name": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "name", + "constraints": { + "presence": false + } + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "keypartone" }, "tableAliases": { "compositetable": "a" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json index d71f0552c6..f24b46859f 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json @@ -17,10 +17,7 @@ }, "filters": { "oneOf": { - "taskid": [ - 1, - 2 - ] + "taskid": [1, 2] } }, "relationships": [ @@ -42,82 +39,78 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": [ - "taskid" - ], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "executorid", - "constraints": { - "presence": false - } - }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "executorid", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false + } + }, + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", "products": "b", "products_tasks": "c" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json index d81c8cf6d0..b746e05c5e 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json @@ -56,48 +56,46 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "c3b91d00cd36c4cc1a347794725b9adbd" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "productname" }, "tableAliases": { "products": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json index 36010bf61d..6a9717f5dd 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json @@ -46,47 +46,45 @@ "tasks": "b", "products": "a" }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", - "primary": ["productid"], - "name": "products", - "schema": { - "productname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "productname", - "constraints": { - "presence": false - } - }, - "productid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "productid", - "constraints": { - "presence": false - } - }, - "tasks": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "tasks", - "relationshipType": "many-to-many", - "fieldName": "taskid", - "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", - "throughFrom": "taskid", - "throughTo": "productid", - "type": "link", - "main": true, - "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products", + "primary": ["productid"], + "name": "products", + "schema": { + "productname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "productname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "productname" - } + "productid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "productid", + "constraints": { + "presence": false + } + }, + "tasks": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "tasks", + "relationshipType": "many-to-many", + "fieldName": "taskid", + "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks", + "throughFrom": "taskid", + "throughTo": "productid", + "type": "link", + "main": true, + "_id": "ca6862d9ba09146dd8a68e3b5b7055a09" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "productname" } } diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json index 01b85f433f..23f56796a9 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json @@ -102,94 +102,92 @@ "extra": { "idFilter": {} }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", - "primary": ["taskid"], - "name": "tasks", - "schema": { - "executorid": { - "type": "number", - "externalType": "integer", - "name": "executorid", - "constraints": { - "presence": false - }, - "autocolumn": true, - "autoReason": "foreign_key" + "table": { + "type": "table", + "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks", + "primary": ["taskid"], + "name": "tasks", + "schema": { + "executorid": { + "type": "number", + "externalType": "integer", + "name": "executorid", + "constraints": { + "presence": false }, - "taskname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "taskname", - "constraints": { - "presence": false - } - }, - "taskid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "taskid", - "constraints": { - "presence": false - } - }, - "completed": { - "type": "boolean", - "externalType": "boolean", - "autocolumn": false, - "name": "completed", - "constraints": { - "presence": false - } - }, - "qaid": { - "type": "number", - "externalType": "integer", - "name": "qaid", - "constraints": { - "presence": false - } - }, - "products": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", - "name": "products", - "relationshipType": "many-to-many", - "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", - "type": "link", - "_id": "c3b91d00cd36c4cc1a347794725b9adbd", - "fieldName": "productid", - "throughFrom": "productid", - "throughTo": "taskid" - }, - "tasksToExecute": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToExecute", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c0f440590bda04f28846242156c1dd60b", - "foreignKey": "executorid", - "fieldName": "personid" - }, - "tasksToQA": { - "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", - "name": "tasksToQA", - "relationshipType": "one-to-many", - "type": "link", - "_id": "c5fdf453a0ba743d58e29491d174c974b", - "foreignKey": "qaid", - "fieldName": "personid" + "autocolumn": true, + "autoReason": "foreign_key" + }, + "taskname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "taskname", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", - "sourceType": "external", - "primaryDisplay": "taskname", - "sql": true, - "views": {} - } + "taskid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "taskid", + "constraints": { + "presence": false + } + }, + "completed": { + "type": "boolean", + "externalType": "boolean", + "autocolumn": false, + "name": "completed", + "constraints": { + "presence": false + } + }, + "qaid": { + "type": "number", + "externalType": "integer", + "name": "qaid", + "constraints": { + "presence": false + } + }, + "products": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products", + "name": "products", + "relationshipType": "many-to-many", + "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks", + "type": "link", + "_id": "c3b91d00cd36c4cc1a347794725b9adbd", + "fieldName": "productid", + "throughFrom": "productid", + "throughTo": "taskid" + }, + "tasksToExecute": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToExecute", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c0f440590bda04f28846242156c1dd60b", + "foreignKey": "executorid", + "fieldName": "personid" + }, + "tasksToQA": { + "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons", + "name": "tasksToQA", + "relationshipType": "one-to-many", + "type": "link", + "_id": "c5fdf453a0ba743d58e29491d174c974b", + "foreignKey": "qaid", + "fieldName": "personid" + } + }, + "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81", + "sourceType": "external", + "primaryDisplay": "taskname", + "sql": true, + "views": {} }, "tableAliases": { "tasks": "a", diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json index 42c2a44335..8171e22c64 100644 --- a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json +++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json @@ -59,123 +59,114 @@ } } }, - "meta": { - "table": { - "type": "table", - "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", - "primary": [ - "personid" - ], - "name": "persons", - "schema": { - "year": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "year", - "constraints": { - "presence": false - } - }, - "firstname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "firstname", - "constraints": { - "presence": false - } - }, - "personid": { - "type": "number", - "externalType": "integer", - "autocolumn": true, - "name": "personid", - "constraints": { - "presence": false - } - }, - "address": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "address", - "constraints": { - "presence": false - } - }, - "age": { - "type": "number", - "externalType": "integer", - "autocolumn": false, - "name": "age", - "constraints": { - "presence": false - } - }, - "type": { - "type": "options", - "externalType": "USER-DEFINED", - "autocolumn": false, - "name": "type", - "constraints": { - "presence": false, - "inclusion": [ - "support", - "designer", - "programmer", - "qa" - ] - } - }, - "city": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "city", - "constraints": { - "presence": false - } - }, - "lastname": { - "type": "string", - "externalType": "character varying", - "autocolumn": false, - "name": "lastname", - "constraints": { - "presence": false - } - }, - "QA": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "QA", - "relationshipType": "many-to-one", - "fieldName": "qaid", - "type": "link", - "main": true, - "_id": "ccb68481c80c34217a4540a2c6c27fe46", - "foreignKey": "personid" - }, - "executor": { - "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", - "name": "executor", - "relationshipType": "many-to-one", - "fieldName": "executorid", - "type": "link", - "main": true, - "_id": "c89530b9770d94bec851e062b5cff3001", - "foreignKey": "personid", - "tableName": "persons" + "table": { + "type": "table", + "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons", + "primary": ["personid"], + "name": "persons", + "schema": { + "year": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "year", + "constraints": { + "presence": false } }, - "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", - "sourceType": "external", - "primaryDisplay": "firstname", - "views": {} - } + "firstname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "firstname", + "constraints": { + "presence": false + } + }, + "personid": { + "type": "number", + "externalType": "integer", + "autocolumn": true, + "name": "personid", + "constraints": { + "presence": false + } + }, + "address": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "address", + "constraints": { + "presence": false + } + }, + "age": { + "type": "number", + "externalType": "integer", + "autocolumn": false, + "name": "age", + "constraints": { + "presence": false + } + }, + "type": { + "type": "options", + "externalType": "USER-DEFINED", + "autocolumn": false, + "name": "type", + "constraints": { + "presence": false, + "inclusion": ["support", "designer", "programmer", "qa"] + } + }, + "city": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "city", + "constraints": { + "presence": false + } + }, + "lastname": { + "type": "string", + "externalType": "character varying", + "autocolumn": false, + "name": "lastname", + "constraints": { + "presence": false + } + }, + "QA": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "QA", + "relationshipType": "many-to-one", + "fieldName": "qaid", + "type": "link", + "main": true, + "_id": "ccb68481c80c34217a4540a2c6c27fe46", + "foreignKey": "personid" + }, + "executor": { + "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks", + "name": "executor", + "relationshipType": "many-to-one", + "fieldName": "executorid", + "type": "link", + "main": true, + "_id": "c89530b9770d94bec851e062b5cff3001", + "foreignKey": "personid", + "tableName": "persons" + } + }, + "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7", + "sourceType": "external", + "primaryDisplay": "firstname", + "views": {} }, "tableAliases": { "persons": "a", "tasks": "b" } -} \ No newline at end of file +} diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index a160bfbb3e..a12b842afc 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -1,8 +1,8 @@ import { Aggregation, CalculationType, - Datasource, DocumentType, + EnrichedQueryJson, FieldType, isLogicalSearchOperator, Operation, @@ -38,7 +38,7 @@ import { generateJunctionTableID } from "../../../../../db/utils" import AliasTables from "../../sqlAlias" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" -import { processRowCountResponse } from "../../utils" +import { enrichQueryJson, processRowCountResponse } from "../../utils" import { dataFilters, helpers, @@ -223,18 +223,18 @@ function reverseUserColumnMapping(rows: Row[], table?: Table) { } function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[] ): Promise function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts: { countTotalRows: true } ): Promise async function runSqlQuery( - json: QueryJson, + json: EnrichedQueryJson, tables: Table[], relationships: RelationshipsJson[], opts?: { countTotalRows?: boolean } @@ -246,7 +246,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT } - const processSQLQuery = async (_: Datasource, json: QueryJson) => { + const processSQLQuery = async (json: EnrichedQueryJson) => { const query = builder._query(json, { disableReturning: true, }) @@ -281,7 +281,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { return processRowCountResponse(response) } else if (Array.isArray(response)) { - return reverseUserColumnMapping(response, json.meta.table) + return reverseUserColumnMapping(response, json.table) } return response } @@ -372,10 +372,7 @@ export async function search( operation: Operation.READ, }, filters: searchFilters, - table, meta: { - table, - tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, resource: { @@ -427,11 +424,13 @@ export async function search( } } + const enrichedRequest = await enrichQueryJson(request) + try { const [rows, totalRows] = await Promise.all([ - runSqlQuery(request, allTables, relationships), + runSqlQuery(enrichedRequest, allTables, relationships), options.countRows - ? runSqlQuery(request, allTables, relationships, { + ? runSqlQuery(enrichedRequest, allTables, relationships, { countTotalRows: true, }) : Promise.resolve(undefined), diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 535709791d..c748940add 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -1,11 +1,13 @@ import { Datasource, DatasourcePlusQueryResponse, + EnrichedQueryJson, Operation, QueryJson, Row, SearchFilters, SqlClient, + Table, } from "@budibase/types" import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" @@ -15,7 +17,6 @@ import { BudibaseInternalDB } from "../../../db/utils" import { dataFilters } from "@budibase/shared-core" type PerformQueryFunction = ( - datasource: Datasource, json: QueryJson ) => Promise @@ -99,7 +100,11 @@ export default class AliasTables { return true } - getAlias(tableName: string) { + getAlias(tableName: string | Table) { + if (typeof tableName === "object") { + tableName = tableName.name + } + if (this.aliases[tableName]) { return this.aliases[tableName] } @@ -177,7 +182,7 @@ export default class AliasTables { } async queryWithAliasing( - json: QueryJson, + json: EnrichedQueryJson, queryFn: PerformQueryFunction ): Promise { const datasourceId = json.endpoint.datasourceId @@ -215,14 +220,7 @@ export default class AliasTables { } json.filters = aliasFilters(json.filters) } - if (json.meta?.table) { - this.getAlias(json.meta.table.name) - } - if (json.meta?.tables) { - Object.keys(json.meta.tables).forEach(tableName => - this.getAlias(tableName) - ) - } + if (json.relationships) { json.relationships = json.relationships.map(relationship => ({ ...relationship, @@ -233,6 +231,12 @@ export default class AliasTables { ]), })) } + + this.getAlias(json.table) + for (const tableName of Object.keys(json.tables)) { + this.getAlias(tableName) + } + // invert and return const invertedTableAliases: Record = {} for (let [key, value] of Object.entries(this.tableAliases)) { @@ -241,7 +245,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse = await queryFn(datasource, json) + let response = await queryFn(json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bded6a7a18..f7646a3a85 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,13 +14,13 @@ import { SqlClient, ArrayOperator, ViewV2, + EnrichedQueryJson, } from "@budibase/types" -import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" -import { docIds, sql } from "@budibase/backend-core" +import { docIds, sql, SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" @@ -73,18 +73,32 @@ export function processRowCountResponse( } } -export async function getDatasourceAndQuery( +export async function enrichQueryJson( json: QueryJson -): Promise { - const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) - const table = datasource.entities?.[json.endpoint.entityId] - if (!json.meta && table) { - json.meta = { - table, +): Promise { + let datasource: Datasource | undefined = undefined + if (json.endpoint.datasourceId !== SQS_DATASOURCE_INTERNAL) { + datasource = await sdk.datasources.get(json.endpoint.datasourceId) + } + + let tables: Record + if (datasource) { + tables = datasource.entities || {} + } else { + tables = {} + for (const table of await sdk.tables.getAllInternalTables()) { + tables[table._id!] = table } } - return makeExternalQuery(datasource, json) + + const table = tables[json.endpoint.entityId] + + return { + table, + tables, + datasource, + ...json, + } } export function cleanExportRows( diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index bdad9c4825..8b0574bd7c 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,5 +1,5 @@ import { Table, Row } from "../documents" -import { QueryJson } from "./search" +import { EnrichedQueryJson } from "./search" export const PASSWORD_REPLACEMENT = "--secret-value--" @@ -207,7 +207,7 @@ export interface DatasourcePlus extends IntegrationBase { // this returns the format of the identifier getBindingIdentifier(): string getStringConcat(parts: string[]): string - query(json: QueryJson): Promise + query(json: EnrichedQueryJson): Promise buildSchema( datasourceId: string, entities: Record diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d6dcb0a2fa..d87472cca9 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -1,5 +1,5 @@ import { Operation } from "./datasources" -import { Row, Table, DocumentType } from "../documents" +import { Row, DocumentType, Table, Datasource } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" import { Aggregation } from "./row" @@ -171,10 +171,7 @@ export interface QueryJson { sort?: SortJson paginate?: PaginationJson body?: Row | Row[] - table?: Table - meta: { - table: Table - tables?: Record + meta?: { renamed?: RenameColumn // can specify something that columns could be prefixed with columnPrefix?: string @@ -186,6 +183,12 @@ export interface QueryJson { tableAliases?: Record } +export interface EnrichedQueryJson extends QueryJson { + table: Table + tables: Record + datasource?: Datasource +} + export interface QueryOptions { disableReturning?: boolean disableBindings?: boolean