From 948ec067d571e49a47bb8a844677df9f4b14fe97 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Nov 2021 13:48:13 +0000 Subject: [PATCH] Updating underlying sql to not use ilike unless in postgres client. --- .../api/controllers/row/ExternalRequest.ts | 2 +- packages/server/src/integrations/base/sql.ts | 421 +++++++++--------- .../src/integrations/microsoftSqlServer.ts | 4 +- 3 files changed, 220 insertions(+), 207 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 23d8deb259..d4e8d475a2 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -342,7 +342,7 @@ module External { table: Table, relationships: RelationshipsJson[] ) { - if (rows[0].read === true) { + if (!rows || rows.length === 0 || rows[0].read === true) { return [] } let finalRows: { [key: string]: Row } = {} diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6c64d5c38f..06a9d0aa10 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -29,222 +29,232 @@ function parseBody(body: any) { return body } -// right now we only do filters on the specific table being queried -function addFilters( - tableName: string, - query: KnexQuery, - filters: SearchFilters | undefined -): KnexQuery { - function iterate( - structure: { [key: string]: any }, - fn: (key: string, value: any) => void - ) { - for (let [key, value] of Object.entries(structure)) { - fn(`${tableName}.${key}`, value) - } +class InternalBuilder { + private readonly client: string + + constructor(client: string) { + this.client = client } - if (!filters) { - return query - } - // if all or specified in filters, then everything is an or - const allOr = filters.allOr - if (filters.oneOf) { - iterate(filters.oneOf, (key, array) => { - const fnc = allOr ? "orWhereIn" : "whereIn" - query = query[fnc](key, array) - }) - } - if (filters.string) { - iterate(filters.string, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "ilike", `${value}%`) - }) - } - if (filters.fuzzy) { - iterate(filters.fuzzy, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "ilike", `%${value}%`) - }) - } - if (filters.range) { - iterate(filters.range, (key, value) => { - if (!value.high || !value.low) { - return + + // right now we only do filters on the specific table being queried + addFilters( + tableName: string, + query: KnexQuery, + filters: SearchFilters | undefined + ): KnexQuery { + function iterate( + structure: { [key: string]: any }, + fn: (key: string, value: any) => void + ) { + for (let [key, value] of Object.entries(structure)) { + fn(`${tableName}.${key}`, value) } - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) - }) - } - if (filters.equal) { - iterate(filters.equal, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc]({ [key]: value }) - }) - } - if (filters.notEqual) { - iterate(filters.notEqual, (key, value) => { - const fnc = allOr ? "orWhereNot" : "whereNot" - query = query[fnc]({ [key]: value }) - }) - } - if (filters.empty) { - iterate(filters.empty, key => { - const fnc = allOr ? "orWhereNull" : "whereNull" - query = query[fnc](key) - }) - } - if (filters.notEmpty) { - iterate(filters.notEmpty, key => { - const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - query = query[fnc](key) - }) - } - return query -} - -function addRelationships( - knex: Knex, - query: KnexQuery, - fields: string | string[], - fromTable: string, - relationships: RelationshipsJson[] | undefined -): KnexQuery { - if (!relationships) { + } + if (!filters) { + return query + } + // if all or specified in filters, then everything is an or + const allOr = filters.allOr + if (filters.oneOf) { + iterate(filters.oneOf, (key, array) => { + const fnc = allOr ? "orWhereIn" : "whereIn" + query = query[fnc](key, array) + }) + } + if (filters.string) { + iterate(filters.string, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + // postgres supports ilike, nothing else does + if (this.client === "pg") { + query = query[fnc](key, "ilike", `${value}%`) + } else { + const rawFnc = `${fnc}Raw` + // @ts-ignore + query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`${value}%`]) + } + }) + } + if (filters.fuzzy) { + iterate(filters.fuzzy, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + // postgres supports ilike, nothing else does + if (this.client === "pg") { + query = query[fnc](key, "ilike", `%${value}%`) + } else { + const rawFnc = `${fnc}Raw` + // @ts-ignore + query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`%${value}%`]) + } + }) + } + if (filters.range) { + iterate(filters.range, (key, value) => { + if (!value.high || !value.low) { + return + } + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) + }) + } + if (filters.equal) { + iterate(filters.equal, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc]({ [key]: value }) + }) + } + if (filters.notEqual) { + iterate(filters.notEqual, (key, value) => { + const fnc = allOr ? "orWhereNot" : "whereNot" + query = query[fnc]({ [key]: value }) + }) + } + if (filters.empty) { + iterate(filters.empty, key => { + const fnc = allOr ? "orWhereNull" : "whereNull" + query = query[fnc](key) + }) + } + if (filters.notEmpty) { + iterate(filters.notEmpty, key => { + const fnc = allOr ? "orWhereNotNull" : "whereNotNull" + query = query[fnc](key) + }) + } return query } - for (let relationship of relationships) { - const from = relationship.from, - to = relationship.to, - toTable = relationship.tableName - if (!relationship.through) { - // @ts-ignore - query = query.leftJoin( - toTable, - `${fromTable}.${from}`, - `${toTable}.${to}` - ) - } else { - const throughTable = relationship.through - const fromPrimary = relationship.fromPrimary - const toPrimary = relationship.toPrimary - query = query + + addRelationships( + knex: Knex, + query: KnexQuery, + fields: string | string[], + fromTable: string, + relationships: RelationshipsJson[] | undefined + ): KnexQuery { + if (!relationships) { + return query + } + for (let relationship of relationships) { + const from = relationship.from, + to = relationship.to, + toTable = relationship.tableName + if (!relationship.through) { // @ts-ignore - .leftJoin( - throughTable, - `${fromTable}.${fromPrimary}`, - `${throughTable}.${from}` + query = query.leftJoin( + toTable, + `${fromTable}.${from}`, + `${toTable}.${to}` ) - .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`) + } else { + const throughTable = relationship.through + const fromPrimary = relationship.fromPrimary + const toPrimary = relationship.toPrimary + query = query + // @ts-ignore + .leftJoin( + throughTable, + `${fromTable}.${fromPrimary}`, + `${throughTable}.${from}` + ) + .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`) + } + } + return query.limit(BASE_LIMIT) + } + + create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, body } = json + let query: KnexQuery = knex(endpoint.entityId) + const parsedBody = parseBody(body) + // make sure no null values in body for creation + for (let [key, value] of Object.entries(parsedBody)) { + if (value == null) { + delete parsedBody[key] + } + } + // mysql can't use returning + if (opts.disableReturning) { + return query.insert(parsedBody) + } else { + return query.insert(parsedBody).returning("*") } } - return query.limit(BASE_LIMIT) -} -function buildCreate( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, body } = json - let query: KnexQuery = knex(endpoint.entityId) - const parsedBody = parseBody(body) - // make sure no null values in body for creation - for (let [key, value] of Object.entries(parsedBody)) { - if (value == null) { - delete parsedBody[key] + read(knex: Knex, json: QueryJson, limit: number): KnexQuery { + let { endpoint, resource, filters, sort, paginate, relationships } = json + const tableName = endpoint.entityId + // select all if not specified + if (!resource) { + resource = { fields: [] } } - } - // mysql can't use returning - if (opts.disableReturning) { - return query.insert(parsedBody) - } else { - return query.insert(parsedBody).returning("*") - } -} - -function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { - let { endpoint, resource, filters, sort, paginate, relationships } = json - const tableName = endpoint.entityId - // select all if not specified - if (!resource) { - resource = { fields: [] } - } - let selectStatement: string | string[] = "*" - // handle select - if (resource.fields && resource.fields.length > 0) { - // select the resources as the format "table.columnName" - this is what is provided - // by the resource builder further up - selectStatement = resource.fields.map(field => `${field} as ${field}`) - } - let foundLimit = limit || BASE_LIMIT - // handle pagination - let foundOffset: number | null = null - if (paginate && paginate.page && paginate.limit) { + let selectStatement: string | string[] = "*" + // handle select + if (resource.fields && resource.fields.length > 0) { + // select the resources as the format "table.columnName" - this is what is provided + // by the resource builder further up + selectStatement = resource.fields.map(field => `${field} as ${field}`) + } + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // start building the query + let query: KnexQuery = knex(tableName).limit(foundLimit) + if (foundOffset) { + query = query.offset(foundOffset) + } + if (sort) { + for (let [key, value] of Object.entries(sort)) { + const direction = value === SortDirection.ASCENDING ? "asc" : "desc" + query = query.orderBy(key, direction) + } + } + query = this.addFilters(tableName, query, filters) // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - foundOffset = offset - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit + let preQuery: KnexQuery = knex({ + // @ts-ignore + [tableName]: query, + }).select(selectStatement) + // handle joins + return this.addRelationships( + knex, + preQuery, + selectStatement, + tableName, + relationships + ) } - // start building the query - let query: KnexQuery = knex(tableName).limit(foundLimit) - if (foundOffset) { - query = query.offset(foundOffset) - } - if (sort) { - for (let [key, value] of Object.entries(sort)) { - const direction = value === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(key, direction) + + update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, body, filters } = json + let query: KnexQuery = knex(endpoint.entityId) + const parsedBody = parseBody(body) + query = this.addFilters(endpoint.entityId, query, filters) + // mysql can't use returning + if (opts.disableReturning) { + return query.update(parsedBody) + } else { + return query.update(parsedBody).returning("*") } } - query = addFilters(tableName, query, filters) - // @ts-ignore - let preQuery: KnexQuery = knex({ - // @ts-ignore - [tableName]: query, - }).select(selectStatement) - // handle joins - return addRelationships( - knex, - preQuery, - selectStatement, - tableName, - relationships - ) -} -function buildUpdate( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, body, filters } = json - let query: KnexQuery = knex(endpoint.entityId) - const parsedBody = parseBody(body) - query = addFilters(endpoint.entityId, query, filters) - // mysql can't use returning - if (opts.disableReturning) { - return query.update(parsedBody) - } else { - return query.update(parsedBody).returning("*") - } -} - -function buildDelete( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, filters } = json - let query: KnexQuery = knex(endpoint.entityId) - query = addFilters(endpoint.entityId, query, filters) - // mysql can't use returning - if (opts.disableReturning) { - return query.delete() - } else { - return query.delete().returning("*") + delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, filters } = json + let query: KnexQuery = knex(endpoint.entityId) + query = this.addFilters(endpoint.entityId, query, filters) + // mysql can't use returning + if (opts.disableReturning) { + return query.delete() + } else { + return query.delete().returning("*") + } } } @@ -266,18 +276,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const sqlClient = this.getSqlClient() const client = knex({ client: sqlClient }) let query + const builder = new InternalBuilder(sqlClient) switch (this._operation(json)) { case Operation.CREATE: - query = buildCreate(client, json, opts) + query = builder.create(client, json, opts) break case Operation.READ: - query = buildRead(client, json, this.limit) + query = builder.read(client, json, this.limit) break case Operation.UPDATE: - query = buildUpdate(client, json, opts) + query = builder.update(client, json, opts) break case Operation.DELETE: - query = buildDelete(client, json, opts) + query = builder.delete(client, json, opts) break case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 7071ce6f75..bcb22f50a9 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -245,7 +245,9 @@ module MSSQLModule { schema, } } - this.tables = tables + const final = finaliseExternalTables(tables) + this.tables = final.tables + this.schemaErrors = final.errors } async read(query: SqlQuery | string) {