MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixing some issues discovered with many to many relationships in SQL, as well as problems uncovered by #3531.

This commit is contained in:
mike12345567 2021-11-24 20:55:03 +00:00
parent bf44c15b74
commit f6ecd423ce
6 changed files with 93 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -197,9 +197,9 @@ module External {
return row
}
function isMany(field: FieldSchema) {
function isOneSide(field: FieldSchema) {
return (
field.relationshipType && field.relationshipType.split("-")[0] === "many"
field.relationshipType && field.relationshipType.split("-")[0] === "one"
)
}
@ -274,25 +274,37 @@ module External {
const linkTable = this.tables[linkTableName]
// @ts-ignore
const linkTablePrimary = linkTable.primary[0]
if (!isMany(field)) {
// one to many
if (isOneSide(field)) {
newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
row[key][0]
)[0]
} else {
}
// many to many
else if (field.through) {
// we're not inserting a doc, will be a bunch of update calls
const isUpdate = !field.through
const thisKey: string = isUpdate
? "id"
: field.throughTo || linkTablePrimary
// @ts-ignore
const otherKey: string = isUpdate
? field.fieldName
: field.throughFrom || tablePrimary
const otherKey: string = field.throughFrom || linkTablePrimary
const thisKey: string = field.throughTo || tablePrimary
row[key].map((relationship: any) => {
// we don't really support composite keys for relationships, this is why [0] is used
manyRelationships.push({
tableId: field.through || field.tableId,
isUpdate,
isUpdate: false,
key: otherKey,
[otherKey]: breakRowIdField(relationship)[0],
// leave the ID for enrichment later
[thisKey]: `{{ literal ${tablePrimary} }}`,
})
})
}
// many to one
else {
const thisKey: string = "id"
// @ts-ignore
const otherKey: string = field.fieldName
row[key].map((relationship: any) => {
manyRelationships.push({
tableId: field.tableId,
isUpdate: true,
key: otherKey,
[thisKey]: breakRowIdField(relationship)[0],
// leave the ID for enrichment later
@ -425,8 +437,8 @@ module External {
)
definition.through = throughTableName
// don't support composite keys for relationships
definition.from = field.throughFrom || table.primary[0]
definition.to = field.throughTo || linkTable.primary[0]
definition.from = field.throughTo || table.primary[0]
definition.to = field.throughFrom || linkTable.primary[0]
definition.fromPrimary = table.primary[0]
definition.toPrimary = linkTable.primary[0]
}
@ -448,24 +460,36 @@ module External {
// make a new request to get the row with all its relationships
// we need this to work out if any relationships need removed
for (let field of Object.values(table.schema)) {
if (field.type !== FieldTypes.LINK || !field.fieldName) {
if (
field.type !== FieldTypes.LINK ||
!field.fieldName ||
isOneSide(field)
) {
continue
}
const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
const tableId = isMany ? field.through : field.tableId
const manyKey = field.throughFrom || primaryKey
const { tableName: relatedTableName } = breakExternalTableId(tableId)
// @ts-ignore
const linkPrimaryKey = this.tables[relatedTableName].primary[0]
const manyKey = field.throughTo || primaryKey
const lookupField = isMany ? primaryKey : field.foreignKey
const fieldName = isMany ? manyKey : field.fieldName
if (!lookupField || !row[lookupField]) {
continue
}
const response = await getDatasourceAndQuery(this.appId, {
endpoint: getEndpoint(tableId, DataSourceOperation.READ),
filters: {
equal: {
[fieldName]: row[primaryKey],
[fieldName]: row[lookupField],
},
},
})
// this is the response from knex if no rows found
const rows = !response[0].read ? response : []
related[fieldName] = { rows, isMany, tableId }
const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
related[storeTo] = { rows, isMany, tableId }
}
return related
}

30
packages/server/src/integrations/base/sql.ts
View File

@ -1,11 +1,11 @@
import { Knex, knex } from "knex"
import {
Operation, PaginationJson,
Operation,
QueryJson,
QueryOptions,
RelationshipsJson,
SearchFilters,
SortDirection, SortJson,
SortDirection,
} from "../../definitions/datasource"
import { isIsoDateString, SqlClients } from "../utils"
import SqlTableQueryBuilder from "./sqlTable"
@ -21,6 +21,9 @@ function parse(input: any) {
if (Array.isArray(input)) {
return JSON.stringify(input)
}
if (input == undefined) {
return null
}
if (typeof input !== "string") {
return input
}
@ -43,7 +46,10 @@ function parseBody(body: any) {
return body
}
function parseFilters(filters: SearchFilters): SearchFilters {
function parseFilters(filters: SearchFilters | undefined): SearchFilters {
if (!filters) {
return {}
}
for (let [key, value] of Object.entries(filters)) {
let parsed
if (typeof value === "object") {
@ -152,21 +158,19 @@ class InternalBuilder {
return query
}
addSorting(
query: KnexQuery,
sort: SortJson | undefined,
paginate: PaginationJson | undefined
): KnexQuery {
addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
let { sort, paginate } = json
if (!sort) {
return query
}
const table = json.meta?.table
for (let [key, value] of Object.entries(sort)) {
const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
query = query.orderBy(key, direction)
query = query.orderBy(`${table?.name}.${key}`, direction)
}
if (this.client === SqlClients.MS_SQL && !sort && paginate?.limit) {
// @ts-ignore
query = query.orderBy(json.meta?.table?.primary[0])
query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
}
return query
}
@ -270,15 +274,15 @@ class InternalBuilder {
}
query = this.addFilters(tableName, query, filters)
// add sorting to pre-query
query = this.addSorting(query, sort, paginate)
query = this.addSorting(query, json)
// @ts-ignore
let preQuery : KnexQuery = knex({
let preQuery: KnexQuery = knex({
// @ts-ignore
[tableName]: query,
}).select(selectStatement)
// have to add after as well (this breaks MS-SQL)
if (this.client !== SqlClients.MS_SQL) {
preQuery = this.addSorting(preQuery, sort, paginate)
preQuery = this.addSorting(preQuery, json)
}
// handle joins
return this.addRelationships(

4
packages/server/src/integrations/index.ts
View File

@ -40,9 +40,9 @@ const INTEGRATIONS = {
}
// optionally add oracle integration if the oracle binary can be installed
if (!(process.arch === 'arm64' && process.platform === 'darwin')) {
if (!(process.arch === "arm64" && process.platform === "darwin")) {
const oracle = require("./oracle")
DEFINITIONS[SourceNames.ORACLE] = oracle.schema
DEFINITIONS[SourceNames.ORACLE] = oracle.schema
INTEGRATIONS[SourceNames.ORACLE] = oracle.integration
}

4
packages/server/src/integrations/mysql.ts
View File

@ -134,7 +134,9 @@ module MySQLModule {
false
)
const tableNames = tablesResp.map(
(obj: any) => obj[`Tables_in_${database}`] || obj[`Tables_in_${database.toLowerCase()}`]
(obj: any) =>
obj[`Tables_in_${database}`] ||
obj[`Tables_in_${database.toLowerCase()}`]
)
for (let tableName of tableNames) {
const primaryKeys = []

29
packages/server/src/integrations/oracle.ts
View File

@ -352,14 +352,23 @@ module OracleModule {
* Knex default returning behaviour does not work with oracle
* Manually add the behaviour for the return column
*/
private addReturning(query: SqlQuery, bindings: BindParameters, returnColumn: string) {
private addReturning(
query: SqlQuery,
bindings: BindParameters,
returnColumn: string
) {
if (bindings instanceof Array) {
bindings.push({ dir: oracledb.BIND_OUT })
query.sql = query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
query.sql =
query.sql + ` returning \"${returnColumn}\" into :${bindings.length}`
}
}
private async internalQuery<T>(query: SqlQuery, returnColum?: string, operation?: string): Promise<Result<T>> {
private async internalQuery<T>(
query: SqlQuery,
returnColum?: string,
operation?: string
): Promise<Result<T>> {
let connection
try {
connection = await this.getConnection()
@ -367,7 +376,10 @@ module OracleModule {
const options: ExecuteOptions = { autoCommit: true }
const bindings: BindParameters = query.bindings || []
if (returnColum && (operation === Operation.CREATE || operation === Operation.UPDATE)) {
if (
returnColum &&
(operation === Operation.CREATE || operation === Operation.UPDATE)
) {
this.addReturning(query, bindings, returnColum)
}
@ -414,14 +426,14 @@ module OracleModule {
return response.rows ? response.rows : []
}
async update(query: SqlQuery | string): Promise<any[]> {
async update(query: SqlQuery | string): Promise<any[]> {
const response = await this.internalQuery(getSqlQuery(query))
return response.rows && response.rows.length
? response.rows
: [{ updated: true }]
}
async delete(query: SqlQuery | string): Promise<any[]> {
async delete(query: SqlQuery | string): Promise<any[]> {
const response = await this.internalQuery(getSqlQuery(query))
return response.rows && response.rows.length
? response.rows
@ -431,8 +443,9 @@ module OracleModule {
async query(json: QueryJson) {
const primaryKeys = json.meta!.table!.primary
const primaryKey = primaryKeys ? primaryKeys[0] : undefined
const queryFn = (query: any, operation: string) => this.internalQuery(query, primaryKey, operation)
const processFn = (response: any) => response.rows ? response.rows : []
const queryFn = (query: any, operation: string) =>
this.internalQuery(query, primaryKey, operation)
const processFn = (response: any) => (response.rows ? response.rows : [])
const output = await this.queryWithReturning(json, queryFn, processFn)
return output
}

8
packages/server/src/integrations/utils.ts
View File

@ -2,7 +2,11 @@ import { SqlQuery } from "../definitions/datasource"
import { Datasource, Table } from "../definitions/common"
import { SourceNames } from "../definitions/datasource"
const { DocumentTypes, SEPARATOR } = require("../db/utils")
const { FieldTypes, BuildSchemaErrors, InvalidColumns } = require("../constants")
const {
FieldTypes,
BuildSchemaErrors,
InvalidColumns,
} = require("../constants")
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
@ -42,7 +46,7 @@ export enum SqlClients {
MS_SQL = "mssql",
POSTGRES = "pg",
MY_SQL = "mysql",
ORACLE = "oracledb"
ORACLE = "oracledb",
}
export function isExternalTable(tableId: string) {