From 0cedd1d57b3ebbe9410db1df693dfefcf4b59d32 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 Nov 2021 14:34:16 +0000 Subject: [PATCH 1/5] First pass end-to-end working --- packages/server/src/integrations/oracle.ts | 263 +++++++++++++++++++-- packages/server/src/integrations/utils.ts | 6 +- 2 files changed, 254 insertions(+), 15 deletions(-) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 5c9f6a1a9a..34f01c8180 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,10 +3,14 @@ import { DatasourceFieldTypes, QueryTypes, SqlQuery, + QueryJson } from "../definitions/datasource" -import { getSqlQuery } from "./utils" -import oracledb, { ExecuteOptions, Result, Connection, ConnectionAttributes } from "oracledb" +import { finaliseExternalTables, getSqlQuery, buildExternalTableId, convertType } from "./utils" +import oracledb, { ExecuteOptions, Result, Connection, ConnectionAttributes, BindParameters } from "oracledb" import Sql from "./base/sql" +import { Table } from "../definitions/common" +import { DatasourcePlus } from "./base/datasourcePlus" +import { FieldTypes } from "../constants" module OracleModule { @@ -22,6 +26,7 @@ module OracleModule { const SCHEMA: Integration = { docs: "https://github.com/oracle/node-oracledb", + plus: true, friendlyName: "Oracle", description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", datasource: { @@ -63,22 +68,237 @@ module OracleModule { }, }, } - class OracleIntegration extends Sql { + + const TYPE_MAP = { + text: FieldTypes.LONGFORM, + blob: FieldTypes.LONGFORM, + enum: FieldTypes.STRING, + varchar: FieldTypes.STRING, + float: FieldTypes.NUMBER, + int: FieldTypes.NUMBER, + numeric: FieldTypes.NUMBER, + bigint: FieldTypes.NUMBER, + mediumint: FieldTypes.NUMBER, + decimal: FieldTypes.NUMBER, + dec: FieldTypes.NUMBER, + double: FieldTypes.NUMBER, + real: FieldTypes.NUMBER, + fixed: FieldTypes.NUMBER, + smallint: FieldTypes.NUMBER, + timestamp: FieldTypes.DATETIME, + date: FieldTypes.DATETIME, + datetime: FieldTypes.DATETIME, + time: FieldTypes.DATETIME, + tinyint: FieldTypes.BOOLEAN, + json: DatasourceFieldTypes.JSON, + } + + /** + * Raw query response + */ + interface ColumnsResponse { + TABLE_NAME: string + COLUMN_NAME: string + DATA_TYPE: string + COLUMN_ID: number + CONSTRAINT_NAME: string | null + CONSTRAINT_TYPE: string | null + R_CONSTRAINT_NAME: string | null + SEARCH_CONDITION: string | null + } + + /** + * An oracle constraint + */ + interface OracleConstraint { + name: string + type: string + relatedConstraintName: string | null + searchCondition: string | null + } + + /** + * An oracle column and it's related constraints + */ + interface OracleColumn { + name: string + type: string + id: number + constraints: {[key: string]: OracleConstraint } + } + + /** + * An oracle table and it's related columns + */ + interface OracleTable { + name: string + columns: {[key: string]: OracleColumn } + } + + const OracleContraintTypes = { + PRIMARY: "P", + NOT_NULL_OR_CHECK: "C", + FOREIGN_KEY: "R", + UNIQUE: "U" + } + + class OracleIntegration extends Sql implements DatasourcePlus { private readonly config: OracleConfig + public tables: Record = {} + public schemaErrors: Record = {} + + private readonly COLUMNS_SQL = ` + SELECT + tabs.table_name, + cols.column_name, + cols.data_type, + cols.column_id, + cons.constraint_name, + cons.constraint_type, + cons.r_constraint_name, + cons.search_condition + FROM + user_tables tabs + JOIN + user_tab_columns cols + ON tabs.table_name = cols.table_name + LEFT JOIN + user_cons_columns col_cons + ON cols.column_name = col_cons.column_name + AND cols.table_name = col_cons.table_name + LEFT JOIN + user_constraints cons + ON col_cons.constraint_name = cons.constraint_name + AND cons.table_name = cols.table_name + WHERE + (cons.status = 'ENABLED' + OR cons.status IS NULL) + ` constructor(config: OracleConfig) { - super("oracle") + super("oracledb") this.config = config } - private query = async (query: SqlQuery): Promise> => { - let connection - try { - connection = await this.getConnection() + /** + * Map the flat tabular columns and constraints data into a nested object + */ + private mapColumns(result: Result): { [key: string]: OracleTable } { + const oracleTables: { [key: string]: OracleTable } = {} - const options : ExecuteOptions = { autoCommit: true } - const result: Result = await connection.execute(query.sql, [], options) + if (result.rows) { + result.rows.forEach(row => { + const tableName = row.TABLE_NAME + const columnName = row.COLUMN_NAME + const dataType = row.DATA_TYPE + const columnId = row.COLUMN_ID + const constraintName = row.CONSTRAINT_NAME + const constraintType = row.CONSTRAINT_TYPE + const relatedConstraintName = row.R_CONSTRAINT_NAME + const searchCondition = row.SEARCH_CONDITION + + let table = oracleTables[tableName] + if (!table) { + table = { + name: tableName, + columns: {} + } + oracleTables[tableName] = table + } + + let column = table.columns[columnName] + if (!column) { + column = { + name: columnName, + type: dataType, + id: columnId, + constraints: {} + } + table.columns[columnName] = column + } + + if (constraintName && constraintType) { + let constraint = column.constraints[constraintName] + if (!constraint) { + constraint = { + name: constraintName, + type: constraintType, + relatedConstraintName: relatedConstraintName, + searchCondition: searchCondition + } + } + column.constraints[constraintName] = constraint + } + }) + } + + return oracleTables + } + + /** + * Fetches the tables from the oracle table and assigns them to the datasource. + * @param {*} datasourceId - datasourceId to fetch + * @param entities - the tables that are to be built + */ + async buildSchema(datasourceId: string, entities: Record) { + const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL }) + const oracleTables = this.mapColumns(columnsResponse) + + const tables: { [key: string]: Table } = {} + + // iterate each table + Object.values(oracleTables).forEach(oracleTable => { + let table = tables[oracleTable.name] + if (!table) { + table = { + _id: buildExternalTableId(datasourceId, oracleTable.name), + primary: [], + name: oracleTable.name, + schema: {}, + } + tables[oracleTable.name] = table + } + + // iterate each column on the table + Object.values(oracleTable.columns) + // match the order of the columns in the db + .sort((c1, c2) => c1.id - c2.id) + .forEach(oracleColumn => { + const columnName = oracleColumn.name + let fieldSchema = table.schema[columnName] + if (!fieldSchema) { + fieldSchema = { + autocolumn: false, + name: columnName, + type: convertType(oracleColumn.type, TYPE_MAP), + } + table.schema[columnName] = fieldSchema + } + + // iterate each constraint on the column + Object.values(oracleColumn.constraints).forEach(oracleConstraint => { + if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { + table.primary!.push(columnName) + } + }) + }) + }) + + const final = finaliseExternalTables(tables, entities) + this.tables = final.tables + this.schemaErrors = final.errors + } + + + private async internalQuery(query: SqlQuery): Promise> { + let connection + try { + connection = await this.getConnection() + + const options: ExecuteOptions = { autoCommit: true } + const bindings: BindParameters = query.bindings || [] + const result: Result = await connection.execute(query.sql, bindings, options) return result } finally { @@ -104,24 +324,39 @@ module OracleModule { } async create(query: SqlQuery | string) { - const response = await this.query(getSqlQuery(query)) + const response = await this.internalQuery(getSqlQuery(query)) return response.rows && response.rows.length ? response.rows : [{ created: true }] } async read(query: SqlQuery | string) { - const response = await this.query(getSqlQuery(query)) + const response = await this.internalQuery(getSqlQuery(query)) return response.rows } async update(query: SqlQuery | string) { - const response = await this.query(getSqlQuery(query)) + const response = await this.internalQuery(getSqlQuery(query)) return response.rows && response.rows.length ? response.rows : [{ updated: true }] } async delete(query: SqlQuery | string) { - const response = await this.query(getSqlQuery(query)) + const response = await this.internalQuery(getSqlQuery(query)) return response.rows && response.rows.length ? response.rows : [{ deleted: true }] } + + async query(json: QueryJson) { + const operation = this._operation(json).toLowerCase() + const input = this._query(json) + if (Array.isArray(input)) { + const responses = [] + for (let query of input) { + responses.push(await this.internalQuery(query)) + } + return responses + } else { + const response = await this.internalQuery(input) + return response.rows && response.rows.length ? response.rows : [{ [operation]: true }] + } + } } module.exports = { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 4d139fa28b..129f3f8d80 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -125,7 +125,7 @@ function copyExistingPropsOver( } export function finaliseExternalTables(tables: { [key: string]: any }, entities: { [key: string]: any }) { - const finalTables: { [key: string]: any } = {} + let finalTables: { [key: string]: any } = {} const errors: { [key: string]: string } = {} for (let [name, table] of Object.entries(tables)) { // make sure every table has a key @@ -136,5 +136,9 @@ export function finaliseExternalTables(tables: { [key: string]: any }, entities: // make sure all previous props have been added back finalTables[name] = copyExistingPropsOver(name, table, entities) } + // sort the tables by name + finalTables = Object.entries(finalTables) + .sort(([a,],[b,]) => a.localeCompare(b)) + .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}); return { tables: finalTables, errors } } From c810bacaf9f636110ef2572cf432ccfbc3293d6a Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 Nov 2021 16:41:00 +0000 Subject: [PATCH 2/5] Datatypes working --- packages/server/src/integrations/oracle.ts | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 34f01c8180..f382cf0aa4 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -69,28 +69,19 @@ module OracleModule { }, } + const UNSUPPORTED_TYPES = [ + "BLOB", + "CLOB", + "NCLOB" + ] + const TYPE_MAP = { - text: FieldTypes.LONGFORM, - blob: FieldTypes.LONGFORM, - enum: FieldTypes.STRING, - varchar: FieldTypes.STRING, - float: FieldTypes.NUMBER, - int: FieldTypes.NUMBER, - numeric: FieldTypes.NUMBER, - bigint: FieldTypes.NUMBER, - mediumint: FieldTypes.NUMBER, - decimal: FieldTypes.NUMBER, - dec: FieldTypes.NUMBER, - double: FieldTypes.NUMBER, - real: FieldTypes.NUMBER, - fixed: FieldTypes.NUMBER, - smallint: FieldTypes.NUMBER, + long: FieldTypes.LONGFORM, + number: FieldTypes.NUMBER, + binary_float: FieldTypes.NUMBER, + binary_double: FieldTypes.NUMBER, timestamp: FieldTypes.DATETIME, date: FieldTypes.DATETIME, - datetime: FieldTypes.DATETIME, - time: FieldTypes.DATETIME, - tinyint: FieldTypes.BOOLEAN, - json: DatasourceFieldTypes.JSON, } /** @@ -236,6 +227,14 @@ module OracleModule { return oracleTables } + private isSupportedColumn(column: OracleColumn) { + if (UNSUPPORTED_TYPES.includes(column.type)) { + return false + } + + return true + } + /** * Fetches the tables from the oracle table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch @@ -262,6 +261,8 @@ module OracleModule { // iterate each column on the table Object.values(oracleTable.columns) + // remove columns that we can't read / save + .filter(oracleColumn => this.isSupportedColumn(oracleColumn)) // match the order of the columns in the db .sort((c1, c2) => c1.id - c2.id) .forEach(oracleColumn => { @@ -290,7 +291,6 @@ module OracleModule { this.schemaErrors = final.errors } - private async internalQuery(query: SqlQuery): Promise> { let connection try { @@ -345,7 +345,7 @@ module OracleModule { async query(json: QueryJson) { const operation = this._operation(json).toLowerCase() - const input = this._query(json) + const input = this._query(json, { disableReturning: true }) if (Array.isArray(input)) { const responses = [] for (let query of input) { From 5d8c90c5f2d42cf53f143dd8738a4678f575d03c Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 18 Nov 2021 10:49:53 +0000 Subject: [PATCH 3/5] Detect auto columns --- packages/server/src/integrations/oracle.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index f382cf0aa4..ded43a59b8 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -91,6 +91,7 @@ module OracleModule { TABLE_NAME: string COLUMN_NAME: string DATA_TYPE: string + DATA_DEFAULT: string | null COLUMN_ID: number CONSTRAINT_NAME: string | null CONSTRAINT_TYPE: string | null @@ -114,6 +115,7 @@ module OracleModule { interface OracleColumn { name: string type: string + default: string | null id: number constraints: {[key: string]: OracleConstraint } } @@ -145,6 +147,7 @@ module OracleModule { tabs.table_name, cols.column_name, cols.data_type, + cols.data_default, cols.column_id, cons.constraint_name, cons.constraint_type, @@ -183,6 +186,7 @@ module OracleModule { const tableName = row.TABLE_NAME const columnName = row.COLUMN_NAME const dataType = row.DATA_TYPE + const dataDefault = row.DATA_DEFAULT const columnId = row.COLUMN_ID const constraintName = row.CONSTRAINT_NAME const constraintType = row.CONSTRAINT_TYPE @@ -203,6 +207,7 @@ module OracleModule { column = { name: columnName, type: dataType, + default: dataDefault, id: columnId, constraints: {} } @@ -235,6 +240,14 @@ module OracleModule { return true } + private isAutoColumn(column: OracleColumn) { + if (column.default && column.default.toLowerCase().includes("nextval")) { + return true + } + + return false + } + /** * Fetches the tables from the oracle table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch @@ -270,7 +283,7 @@ module OracleModule { let fieldSchema = table.schema[columnName] if (!fieldSchema) { fieldSchema = { - autocolumn: false, + autocolumn: this.isAutoColumn(oracleColumn), name: columnName, type: convertType(oracleColumn.type, TYPE_MAP), } From 345490fed3b7afcabd89c2f77bf5acbf6a153068 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 18 Nov 2021 12:05:30 +0000 Subject: [PATCH 4/5] Tests --- packages/server/__mocks__/oracledb.ts | 31 ++++++ .../src/integrations/tests/oracle.spec.js | 94 +++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 packages/server/__mocks__/oracledb.ts create mode 100644 packages/server/src/integrations/tests/oracle.spec.js diff --git a/packages/server/__mocks__/oracledb.ts b/packages/server/__mocks__/oracledb.ts new file mode 100644 index 0000000000..fd19845eee --- /dev/null +++ b/packages/server/__mocks__/oracledb.ts @@ -0,0 +1,31 @@ +module OracleDbMock { + // mock execute + const execute = jest.fn(() => ({ + rows: [ + { + a: "string", + b: 1, + }, + ], + })) + + const close = jest.fn() + + // mock connection + function Connection() {} + Connection.prototype.execute = execute + Connection.prototype.close = close + + // mock oracledb + const oracleDb: any = {} + oracleDb.getConnection = jest.fn(() => { + // @ts-ignore + return new Connection() + }) + + // expose mocks + oracleDb.executeMock = execute + oracleDb.closeMock = close + + module.exports = oracleDb +} diff --git a/packages/server/src/integrations/tests/oracle.spec.js b/packages/server/src/integrations/tests/oracle.spec.js new file mode 100644 index 0000000000..77f0525090 --- /dev/null +++ b/packages/server/src/integrations/tests/oracle.spec.js @@ -0,0 +1,94 @@ +const oracledb = require("oracledb") +const OracleIntegration = require("../oracle") +jest.mock("oracledb") + +class TestConfiguration { + constructor(config = {}) { + this.integration = new OracleIntegration.integration(config) + } +} + +const options = { autoCommit: true } + +describe("Oracle Integration", () => { + let config + + beforeEach(() => { + jest.clearAllMocks() + config = new TestConfiguration() + }) + + afterEach(() => { + expect(oracledb.closeMock).toHaveBeenCalled() + expect(oracledb.closeMock).toHaveBeenCalledTimes(1) + }) + + it("calls the create method with the correct params", async () => { + const sql = "insert into users (name, age) values ('Joe', 123);" + await config.integration.create({ + sql + }) + expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + it("calls the read method with the correct params", async () => { + const sql = "select * from users;" + await config.integration.read({ + sql + }) + expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + it("calls the update method with the correct params", async () => { + const sql = "update table users set name = 'test';" + const response = await config.integration.update({ + sql + }) + expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + it("calls the delete method with the correct params", async () => { + const sql = "delete from users where name = 'todelete';" + await config.integration.delete({ + sql + }) + expect(oracledb.executeMock).toHaveBeenCalledWith(sql, [], options) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + describe("no rows returned", () => { + beforeEach(() => { + oracledb.executeMock.mockImplementation(() => ({ rows: [] })) + }) + + it("returns the correct response when the create response has no rows", async () => { + const sql = "insert into users (name, age) values ('Joe', 123);" + const response = await config.integration.create({ + sql + }) + expect(response).toEqual([{ created: true }]) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + it("returns the correct response when the update response has no rows", async () => { + const sql = "update table users set name = 'test';" + const response = await config.integration.update({ + sql + }) + expect(response).toEqual([{ updated: true }]) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + + it("returns the correct response when the delete response has no rows", async () => { + const sql = "delete from users where name = 'todelete';" + const response = await config.integration.delete({ + sql + }) + expect(response).toEqual([{ deleted: true }]) + expect(oracledb.executeMock).toHaveBeenCalledTimes(1) + }) + }) +}) \ No newline at end of file From b0df7fb28fad1e60e9a9f8687d7950974ae8d39c Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Thu, 18 Nov 2021 13:35:22 +0000 Subject: [PATCH 5/5] Boolean support and linting --- .../api/controllers/row/ExternalRequest.ts | 7 +- packages/server/src/integrations/base/sql.ts | 4 +- .../server/src/integrations/base/sqlTable.ts | 44 ++-- .../server/src/integrations/base/utils.ts | 5 +- packages/server/src/integrations/oracle.ts | 189 ++++++++++++------ packages/server/src/integrations/utils.ts | 9 +- 6 files changed, 176 insertions(+), 82 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index f538e01f73..23d8deb259 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -226,7 +226,12 @@ module External { manyRelationships: ManyRelationship[] = [] for (let [key, field] of Object.entries(table.schema)) { // if set already, or not set just skip it - if (row[key] == null || newRow[key] || field.autocolumn || field.type === FieldTypes.FORMULA) { + if ( + row[key] == null || + newRow[key] || + field.autocolumn || + field.type === FieldTypes.FORMULA + ) { continue } // if its an empty string then it means return the column to null (if possible) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index a51f57ee47..53d43b6260 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -279,7 +279,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { case Operation.DELETE: query = buildDelete(client, json, opts) break - case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: case Operation.DELETE_TABLE: + case Operation.CREATE_TABLE: + case Operation.UPDATE_TABLE: + case Operation.DELETE_TABLE: return this._tableQuery(json) default: throw `Operation type is not supported by SQL query builder` diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index e5249dfe7c..974f395063 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -6,7 +6,12 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder const { FieldTypes, RelationshipTypes } = require("../../constants") -function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record, oldTable: null | Table = null) { +function generateSchema( + schema: CreateTableBuilder, + table: Table, + tables: Record, + oldTable: null | Table = null +) { let primaryKey = table && table.primary ? table.primary[0] : null const columns = Object.values(table.schema) // all columns in a junction table will be meta @@ -19,17 +24,21 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record schema.primary(metaCols.map(col => col.name)) } - // check if any columns need added const foreignKeys = Object.values(table.schema).map(col => col.foreignKey) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct const oldColumn = oldTable ? oldTable.schema[key] : null - if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) { + if ( + (oldColumn && oldColumn.type === column.type) || + (primaryKey === key && !isJunction) + ) { continue } switch (column.type) { - case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: + case FieldTypes.STRING: + case FieldTypes.OPTIONS: + case FieldTypes.LONGFORM: schema.string(key) break case FieldTypes.NUMBER: @@ -67,7 +76,9 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record throw "Referenced table doesn't exist" } schema.integer(column.foreignKey).unsigned() - schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`) + schema + .foreign(column.foreignKey) + .references(`${tableName}.${relatedTable.primary[0]}`) } break } @@ -76,7 +87,10 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record // need to check if any columns have been deleted if (oldTable) { const deletedColumns = Object.entries(oldTable.schema) - .filter(([key, schema]) => schema.type !== FieldTypes.LINK && table.schema[key] == null) + .filter( + ([key, schema]) => + schema.type !== FieldTypes.LINK && table.schema[key] == null + ) .map(([key]) => key) deletedColumns.forEach(key => { if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) { @@ -92,7 +106,7 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record function buildCreateTable( knex: Knex, table: Table, - tables: Record, + tables: Record ): SchemaBuilder { return knex.schema.createTable(table.name, schema => { generateSchema(schema, table, tables) @@ -103,17 +117,14 @@ function buildUpdateTable( knex: Knex, table: Table, tables: Record, - oldTable: Table, + oldTable: Table ): SchemaBuilder { return knex.schema.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable) }) } -function buildDeleteTable( - knex: Knex, - table: Table, -): SchemaBuilder { +function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder { return knex.schema.dropTable(table.name) } @@ -151,7 +162,12 @@ class SqlTableQueryBuilder { if (!json.meta || !json.meta.table) { throw "Must specify old table for update" } - query = buildUpdateTable(client, json.table, json.meta.tables, json.meta.table) + query = buildUpdateTable( + client, + json.table, + json.meta.tables, + json.meta.table + ) break case Operation.DELETE_TABLE: query = buildDeleteTable(client, json.table) @@ -164,4 +180,4 @@ class SqlTableQueryBuilder { } export default SqlTableQueryBuilder -module.exports = SqlTableQueryBuilder \ No newline at end of file +module.exports = SqlTableQueryBuilder diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts index 5757232bc7..086912b920 100644 --- a/packages/server/src/integrations/base/utils.ts +++ b/packages/server/src/integrations/base/utils.ts @@ -4,7 +4,10 @@ import { Datasource } from "../../definitions/common" module DatasourceUtils { const { integrations } = require("../index") - export async function makeExternalQuery(datasource: Datasource, json: QueryJson) { + export async function makeExternalQuery( + datasource: Datasource, + json: QueryJson + ) { const Integration = integrations[datasource.source] // query is the opinionated function if (Integration.prototype.query) { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index ded43a59b8..336feac91f 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -3,18 +3,28 @@ import { DatasourceFieldTypes, QueryTypes, SqlQuery, - QueryJson + QueryJson, } from "../definitions/datasource" -import { finaliseExternalTables, getSqlQuery, buildExternalTableId, convertType } from "./utils" -import oracledb, { ExecuteOptions, Result, Connection, ConnectionAttributes, BindParameters } from "oracledb" +import { + finaliseExternalTables, + getSqlQuery, + buildExternalTableId, + convertType, +} from "./utils" +import oracledb, { + ExecuteOptions, + Result, + Connection, + ConnectionAttributes, + BindParameters, +} from "oracledb" import Sql from "./base/sql" import { Table } from "../definitions/common" import { DatasourcePlus } from "./base/datasourcePlus" import { FieldTypes } from "../constants" module OracleModule { - - oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT; + oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT interface OracleConfig { host: string @@ -28,7 +38,8 @@ module OracleModule { docs: "https://github.com/oracle/node-oracledb", plus: true, friendlyName: "Oracle", - description: "Oracle Database is an object-relational database management system developed by Oracle Corporation", + description: + "Oracle Database is an object-relational database management system developed by Oracle Corporation", datasource: { host: { type: DatasourceFieldTypes.STRING, @@ -51,7 +62,7 @@ module OracleModule { password: { type: DatasourceFieldTypes.PASSWORD, required: true, - } + }, }, query: { create: { @@ -69,11 +80,7 @@ module OracleModule { }, } - const UNSUPPORTED_TYPES = [ - "BLOB", - "CLOB", - "NCLOB" - ] + const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"] const TYPE_MAP = { long: FieldTypes.LONGFORM, @@ -104,7 +111,7 @@ module OracleModule { */ interface OracleConstraint { name: string - type: string + type: string relatedConstraintName: string | null searchCondition: string | null } @@ -117,7 +124,7 @@ module OracleModule { type: string default: string | null id: number - constraints: {[key: string]: OracleConstraint } + constraints: { [key: string]: OracleConstraint } } /** @@ -125,18 +132,17 @@ module OracleModule { */ interface OracleTable { name: string - columns: {[key: string]: OracleColumn } + columns: { [key: string]: OracleColumn } } const OracleContraintTypes = { PRIMARY: "P", NOT_NULL_OR_CHECK: "C", FOREIGN_KEY: "R", - UNIQUE: "U" + UNIQUE: "U", } class OracleIntegration extends Sql implements DatasourcePlus { - private readonly config: OracleConfig public tables: Record = {} @@ -176,9 +182,11 @@ module OracleModule { } /** - * Map the flat tabular columns and constraints data into a nested object + * Map the flat tabular columns and constraints data into a nested object */ - private mapColumns(result: Result): { [key: string]: OracleTable } { + private mapColumns(result: Result): { + [key: string]: OracleTable + } { const oracleTables: { [key: string]: OracleTable } = {} if (result.rows) { @@ -197,7 +205,7 @@ module OracleModule { if (!table) { table = { name: tableName, - columns: {} + columns: {}, } oracleTables[tableName] = table } @@ -207,9 +215,9 @@ module OracleModule { column = { name: columnName, type: dataType, - default: dataDefault, + default: dataDefault, id: columnId, - constraints: {} + constraints: {}, } table.columns[columnName] = column } @@ -221,7 +229,7 @@ module OracleModule { name: constraintName, type: constraintType, relatedConstraintName: relatedConstraintName, - searchCondition: searchCondition + searchCondition: searchCondition, } } column.constraints[constraintName] = constraint @@ -248,13 +256,54 @@ module OracleModule { return false } + /** + * No native boolean in oracle. Best we can do is to check if a manual 1 or 0 number constraint has been set up + * This matches the default behaviour for generating DDL used in knex. + */ + private isBooleanType(column: OracleColumn): boolean { + if ( + column.type.toLowerCase() === "number" && + Object.values(column.constraints).filter(c => { + if ( + c.type === OracleContraintTypes.NOT_NULL_OR_CHECK && + c.searchCondition + ) { + const condition = c.searchCondition + .replace(/\s/g, "") // remove spaces + .replace(/[']+/g, "") // remove quotes + if ( + condition.includes("in(0,1)") || + condition.includes("in(1,0)") + ) { + return true + } + } + return false + }).length > 0 + ) { + return true + } + + return false + } + + private internalConvertType(column: OracleColumn): string { + if (this.isBooleanType(column)) { + return FieldTypes.BOOLEAN + } + + return convertType(column.type, TYPE_MAP) + } + /** * Fetches the tables from the oracle table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch * @param entities - the tables that are to be built - */ + */ async buildSchema(datasourceId: string, entities: Record) { - const columnsResponse = await this.internalQuery({ sql: this.COLUMNS_SQL }) + const columnsResponse = await this.internalQuery({ + sql: this.COLUMNS_SQL, + }) const oracleTables = this.mapColumns(columnsResponse) const tables: { [key: string]: Table } = {} @@ -274,29 +323,31 @@ module OracleModule { // iterate each column on the table Object.values(oracleTable.columns) - // remove columns that we can't read / save - .filter(oracleColumn => this.isSupportedColumn(oracleColumn)) - // match the order of the columns in the db - .sort((c1, c2) => c1.id - c2.id) - .forEach(oracleColumn => { - const columnName = oracleColumn.name - let fieldSchema = table.schema[columnName] - if (!fieldSchema) { - fieldSchema = { - autocolumn: this.isAutoColumn(oracleColumn), - name: columnName, - type: convertType(oracleColumn.type, TYPE_MAP), + // remove columns that we can't read / save + .filter(oracleColumn => this.isSupportedColumn(oracleColumn)) + // match the order of the columns in the db + .sort((c1, c2) => c1.id - c2.id) + .forEach(oracleColumn => { + const columnName = oracleColumn.name + let fieldSchema = table.schema[columnName] + if (!fieldSchema) { + fieldSchema = { + autocolumn: this.isAutoColumn(oracleColumn), + name: columnName, + type: this.internalConvertType(oracleColumn), + } + table.schema[columnName] = fieldSchema } - table.schema[columnName] = fieldSchema - } - // iterate each constraint on the column - Object.values(oracleColumn.constraints).forEach(oracleConstraint => { - if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { - table.primary!.push(columnName) - } + // iterate each constraint on the column + Object.values(oracleColumn.constraints).forEach( + oracleConstraint => { + if (oracleConstraint.type === OracleContraintTypes.PRIMARY) { + table.primary!.push(columnName) + } + } + ) }) - }) }) const final = finaliseExternalTables(tables, entities) @@ -305,40 +356,48 @@ module OracleModule { } private async internalQuery(query: SqlQuery): Promise> { - let connection - try { - connection = await this.getConnection() + let connection + try { + connection = await this.getConnection() const options: ExecuteOptions = { autoCommit: true } const bindings: BindParameters = query.bindings || [] - const result: Result = await connection.execute(query.sql, bindings, options) + const result: Result = await connection.execute( + query.sql, + bindings, + options + ) return result } finally { - if (connection) { - try { - await connection.close(); - } catch (err) { - console.error(err); - } - } + if (connection) { + try { + await connection.close() + } catch (err) { + console.error(err) + } + } } } private getConnection = async (): Promise => { //connectString : "(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = localhost)(PORT = 1521))(CONNECT_DATA =(SID= ORCL)))" - const connectString = `${this.config.host}:${this.config.port || 1521}/${this.config.database}` + const connectString = `${this.config.host}:${this.config.port || 1521}/${ + this.config.database + }` const attributes: ConnectionAttributes = { user: this.config.user, password: this.config.user, connectString, } - return oracledb.getConnection(attributes); + return oracledb.getConnection(attributes) } async create(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ created: true }] + return response.rows && response.rows.length + ? response.rows + : [{ created: true }] } async read(query: SqlQuery | string) { @@ -348,12 +407,16 @@ module OracleModule { async update(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ updated: true }] + return response.rows && response.rows.length + ? response.rows + : [{ updated: true }] } async delete(query: SqlQuery | string) { const response = await this.internalQuery(getSqlQuery(query)) - return response.rows && response.rows.length ? response.rows : [{ deleted: true }] + return response.rows && response.rows.length + ? response.rows + : [{ deleted: true }] } async query(json: QueryJson) { @@ -367,7 +430,9 @@ module OracleModule { return responses } else { const response = await this.internalQuery(input) - return response.rows && response.rows.length ? response.rows : [{ [operation]: true }] + return response.rows && response.rows.length + ? response.rows + : [{ [operation]: true }] } } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 129f3f8d80..f1cc9eab9f 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -124,7 +124,10 @@ function copyExistingPropsOver( return table } -export function finaliseExternalTables(tables: { [key: string]: any }, entities: { [key: string]: any }) { +export function finaliseExternalTables( + tables: { [key: string]: any }, + entities: { [key: string]: any } +) { let finalTables: { [key: string]: any } = {} const errors: { [key: string]: string } = {} for (let [name, table] of Object.entries(tables)) { @@ -138,7 +141,7 @@ export function finaliseExternalTables(tables: { [key: string]: any }, entities: } // sort the tables by name finalTables = Object.entries(finalTables) - .sort(([a,],[b,]) => a.localeCompare(b)) - .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}); + .sort(([a], [b]) => a.localeCompare(b)) + .reduce((r, [k, v]) => ({ ...r, [k]: v }), {}) return { tables: finalTables, errors } }