From cb3495c831288a2f307697fdd70babd908ca4948 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 18 Nov 2021 13:35:22 +0000 Subject: [PATCH] Boolean support and linting --- .../api/controllers/row/ExternalRequest.ts | 7 +- packages/server/src/integrations/base/sql.ts | 4 +- .../server/src/integrations/base/sqlTable.ts | 44 ++-- .../server/src/integrations/base/utils.ts | 5 +- packages/server/src/integrations/oracle.ts | 189 ++++++++++++------ packages/server/src/integrations/utils.ts | 9 +- 6 files changed, 176 insertions(+), 82 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index f538e01f73..23d8deb259 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -226,7 +226,12 @@ module External { manyRelationships: ManyRelationship[] = [] for (let [key, field] of Object.entries(table.schema)) { // if set already, or not set just skip it - if (row[key] == null || newRow[key] || field.autocolumn || field.type === FieldTypes.FORMULA) { + if ( + row[key] == null || + newRow[key] || + field.autocolumn || + field.type === FieldTypes.FORMULA + ) { continue } // if its an empty string then it means return the column to null (if possible) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index a51f57ee47..53d43b6260 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -279,7 +279,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { case Operation.DELETE: query = buildDelete(client, json, opts) break - case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: case Operation.DELETE_TABLE: + case Operation.CREATE_TABLE: + case Operation.UPDATE_TABLE: + case Operation.DELETE_TABLE: return this._tableQuery(json) default: throw `Operation type is not supported by SQL query builder` diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index e5249dfe7c..974f395063 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -6,7 +6,12 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder const { FieldTypes, RelationshipTypes } = require("../../constants") -function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record, oldTable: null | Table = null) { +function generateSchema( + schema: CreateTableBuilder, + table: Table, + tables: Record, + oldTable: null | Table = null +) { let primaryKey = table && table.primary ? table.primary[0] : null const columns = Object.values(table.schema) // all columns in a junction table will be meta @@ -19,17 +24,21 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record schema.primary(metaCols.map(col => col.name)) } - // check if any columns need added const foreignKeys = Object.values(table.schema).map(col => col.foreignKey) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct const oldColumn = oldTable ? oldTable.schema[key] : null - if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) { + if ( + (oldColumn && oldColumn.type === column.type) || + (primaryKey === key && !isJunction) + ) { continue } switch (column.type) { - case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: + case FieldTypes.STRING: + case FieldTypes.OPTIONS: + case FieldTypes.LONGFORM: schema.string(key) break case FieldTypes.NUMBER: @@ -67,7 +76,9 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record throw "Referenced table doesn't exist" } schema.integer(column.foreignKey).unsigned() - schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`) + schema + .foreign(column.foreignKey) + .references(`${tableName}.${relatedTable.primary[0]}`) } break } @@ -76,7 +87,10 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record // need to check if any columns have been deleted if (oldTable) { const deletedColumns = Object.entries(oldTable.schema) - .filter(([key, schema]) => schema.type !== FieldTypes.LINK && table.schema[key] == null) + .filter( + ([key, schema]) => + schema.type !== FieldTypes.LINK && table.schema[key] == null + ) .map(([key]) => key) deletedColumns.forEach(key => { if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) { @@ -92,7 +106,7 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record function buildCreateTable( knex: Knex, table: Table, - tables: Record, + tables: Record ): SchemaBuilder { return knex.schema.createTable(table.name, schema => { generateSchema(schema, table, tables) @@ -103,17 +117,14 @@ function buildUpdateTable( knex: Knex, table: Table, tables: Record, - oldTable: Table, + oldTable: Table ): SchemaBuilder { return knex.schema.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable) }) } -function buildDeleteTable( - knex: Knex, - table: Table, -): SchemaBuilder { +function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder { return knex.schema.dropTable(table.name) } @@ -151,7 +162,12 @@ class SqlTableQueryBuilder { if (!json.meta || !json.meta.table) { throw "Must specify old table for update" } - query = buildUpdateTable(client, json.table, json.meta.tables, json.meta.table) + query = buildUpdateTable( + client, + json.table, + json.meta.tables, + json.meta.table + ) break case Operation.DELETE_TABLE: query = buildDeleteTable(client, json.table) @@ -164,4 +180,4 @@ class SqlTableQueryBuilder { } export default SqlTableQueryBuilder -module.exports = SqlTableQueryBuilder \ No newline at end of file +module.exports = SqlTableQueryBuilder diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts index 5757232bc7..086912b920 100644 --- a/packages/server/src/integrations/base/utils.ts +++ b/packages/server/src/integrations/base/utils.ts @@ -4,7 +4,10 @@ import { Datasource } from "../../definitions/common" module DatasourceUtils { const { integrations } = require("../index") - export async function makeExternalQuery(datasource: Datasource, json: QueryJson) { + export async function makeExternalQuery( + datasource: Datasource, + json: QueryJson + ) { const Integration = integrations[datasource.source] // query is the opinionated function if (Integration.prototype.query) { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index ded43a59b8..336feac91f 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,18 +3,28 @@ import { DatasourceFieldTypes, QueryTypes, SqlQuery, - QueryJson + QueryJson, } from "../definitions/datasource" -import { finaliseExternalTables, getSqlQuery, buildExternalTableId, convertType } from "./utils" -import oracledb, { ExecuteOptions, Result, Connection, ConnectionAttributes, BindParameters } from "oracledb" +import { + finaliseExternalTables, + getSqlQuery, + buildExternalTableId, + convertType, +} from "./utils" +import oracledb, { + ExecuteOptions, + Result, + Connection, + ConnectionAttributes, + BindParameters, +} from "oracledb" import Sql from "./base/sql" import { Table } from "../definitions/common" import { DatasourcePlus } from "./base/datasourcePlus" import { FieldTypes } from "../constants" module OracleModule { - - oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT; + oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT interface OracleConfig { host: string @@ -28,7 +38,8 @@ module OracleModule { docs: "https://github.com/oracle/node-oracledb", plus: true, friendlyName: "Oracle", - description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", + description: + "Oracle Database is an object-relational database management system developed by Oracle Corporation", datasource: { host: { type: DatasourceFieldTypes.STRING, @@ -51,7 +62,7 @@ module OracleModule { password: { type: DatasourceFieldTypes.PASSWORD, required: true, - } + }, }, query: { create: { @@ -69,11 +80,7 @@ module OracleModule { }, } - const UNSUPPORTED_TYPES = [ - "BLOB", - "CLOB", - "NCLOB" - ] + const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"] const TYPE_MAP = { long: FieldTypes.LONGFORM, @@ -104,7 +111,7 @@ module OracleModule { */ interface OracleConstraint { name: string - type: string + type: string relatedConstraintName: string | null searchCondition: string | null } @@ -117,7 +124,7 @@ module OracleModule { type: string default: string | null id: number - constraints: {[key: string]: OracleConstraint } + constraints: { [key: string]: OracleConstraint } } /** @@ -125,18 +132,17 @@ module OracleModule { */ interface OracleTable { name: string - columns: {[key: string]: OracleColumn } + columns: { [key: string]: OracleColumn } } const OracleContraintTypes = { PRIMARY: "P", NOT_NULL_OR_CHECK: "C", FOREIGN_KEY: "R", - UNIQUE: "U" + UNIQUE: "U", } class OracleIntegration extends Sql implements DatasourcePlus { - private readonly config: OracleConfig public tables: Record = {} @@ -176,9 +182,11 @@ module OracleModule { } /** - * Map the flat tabular columns and constraints data into a nested object + * Map the flat tabular columns and constraints data into a nested object */ - private mapColumns(result: Result): { [key: string]: OracleTable } { + private mapColumns(result: Result): { + [key: string]: OracleTable + } { const oracleTables: { [key: string]: OracleTable } = {} if (result.rows) { @@ -197,7 +205,7 @@ module OracleModule { if (!table) { table = { name: tableName, - columns: {} + columns: {}, } oracleTables[tableName] = table } @@ -207,9 +215,9 @@ module OracleModule { column = { name: columnName, type: dataType, - default: dataDefault, + default: dataDefault, id: columnId, - constraints: {} + constraints: {}, } table.columns[columnName] = column } @@ -221,7 +229,7 @@ module OracleModule { name: constraintName, type: constraintType, relatedConstraintName: relatedConstraintName, - searchCondition: searchCondition + searchCondition: searchCondition, } } column.constraints[constraintName] = constraint @@ -248,13 +256,54 @@ module OracleModule { return false } + /** + * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up + * This matches the default behaviour for generating DDL used in knex. + */ + private isBooleanType(column: OracleColumn): boolean { + if ( + column.type.toLowerCase() === "number" && + Object.values(column.constraints).filter(c => { + if ( + c.type === OracleContraintTypes.NOT_NULL_OR_CHECK && + c.searchCondition + ) { + const condition = c.searchCondition + .replace(/\s/g, "") // remove spaces + .replace(/[']+/g, "") // remove quotes + if ( + condition.includes("in(0,1)") || + condition.includes("in(1,0)") + ) { + return true + } + } + return false + }).length > 0 + ) { + return true + } + + return false + } + + private internalConvertType(column: OracleColumn): string { + if (this.isBooleanType(column)) { + return FieldTypes.BOOLEAN + } + + return convertType(column.type, TYPE_MAP) + } + /** * Fetches the tables from the oracle table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch * @param entities - the tables that are to be built - */ + */ async buildSchema(datasourceId: string, entities: Record) { - const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL }) + const columnsResponse = await this.internalQuery({ + sql: this.COLUMNS_SQL, + }) const oracleTables = this.mapColumns(columnsResponse) const tables: { [key: string]: Table } = {} @@ -274,29 +323,31 @@ module OracleModule { // iterate each column on the table Object.values(oracleTable.columns) - // remove columns that we can't read / save - .filter(oracleColumn => this.isSupportedColumn(oracleColumn)) - // match the order of the columns in the db - .sort((c1, c2) => c1.id - c2.id) - .forEach(oracleColumn => { - const columnName = oracleColumn.name - let fieldSchema = table.schema[columnName] - if (!fieldSchema) { - fieldSchema = { - autocolumn: this.isAutoColumn(oracleColumn), - name: columnName, - type: convertType(oracleColumn.type, TYPE_MAP), + // remove columns that we can't read / save + .filter(oracleColumn => this.isSupportedColumn(oracleColumn)) + // match the order of the columns in the db + .sort((c1, c2) => c1.id - c2.id) + .forEach(oracleColumn => { + const columnName = oracleColumn.name + let fieldSchema = table.schema[columnName] + if (!fieldSchema) { + fieldSchema = { + autocolumn: this.isAutoColumn(oracleColumn), + name: columnName, + type: this.internalConvertType(oracleColumn), + } + table.schema[columnName] = fieldSchema } - table.schema[columnName] = fieldSchema - } - // iterate each constraint on the column - Object.values(oracleColumn.constraints).forEach(oracleConstraint => { - if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { - table.primary!.push(columnName) - } + // iterate each constraint on the column + Object.values(oracleColumn.constraints).forEach( + oracleConstraint => { + if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { + table.primary!.push(columnName) + } + } + ) }) - }) }) const final = finaliseExternalTables(tables, entities) @@ -305,40 +356,48 @@ module OracleModule { } private async internalQuery(query: SqlQuery): Promise> { - let connection - try { - connection = await this.getConnection() + let connection + try { + connection = await this.getConnection() const options: ExecuteOptions = { autoCommit: true } const bindings: BindParameters = query.bindings || [] - const result: Result = await connection.execute(query.sql, bindings, options) + const result: Result = await connection.execute( + query.sql, + bindings, + options + ) return result } finally { - if (connection) { - try { - await connection.close(); - } catch (err) { - console.error(err); - } - } + if (connection) { + try { + await connection.close() + } catch (err) { + console.error(err) + } + } } } private getConnection = async (): Promise => { //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))" - const connectString = `${this.config.host}:${this.config.port || 1521}/${this.config.database}` + const connectString = `${this.config.host}:${this.config.port || 1521}/${ + this.config.database + }` const attributes: ConnectionAttributes = { user: this.config.user, password: this.config.user, connectString, } - return oracledb.getConnection(attributes); + return oracledb.getConnection(attributes) } async create(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ created: true }] + return response.rows && response.rows.length + ? response.rows + : [{ created: true }] } async read(query: SqlQuery | string) { @@ -348,12 +407,16 @@ module OracleModule { async update(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ updated: true }] + return response.rows && response.rows.length + ? response.rows + : [{ updated: true }] } async delete(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ deleted: true }] + return response.rows && response.rows.length + ? response.rows + : [{ deleted: true }] } async query(json: QueryJson) { @@ -367,7 +430,9 @@ module OracleModule { return responses } else { const response = await this.internalQuery(input) - return response.rows && response.rows.length ? response.rows : [{ [operation]: true }] + return response.rows && response.rows.length + ? response.rows + : [{ [operation]: true }] } } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 129f3f8d80..f1cc9eab9f 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -124,7 +124,10 @@ function copyExistingPropsOver( return table } -export function finaliseExternalTables(tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function finaliseExternalTables( + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { let finalTables: { [key: string]: any } = {} const errors: { [key: string]: string } = {} for (let [name, table] of Object.entries(tables)) { @@ -138,7 +141,7 @@ export function finaliseExternalTables(tables: { [key: string]: any }, entities: } // sort the tables by name finalTables = Object.entries(finalTables) - .sort(([a,],[b,]) => a.localeCompare(b)) - .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}); + .sort(([a], [b]) => a.localeCompare(b)) + .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}) return { tables: finalTables, errors } }