From 269ad4ee66d9c47c225e601a3f147e24df1f90a9 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:01:36 +0000 Subject: [PATCH] Support enum types in PostgreSQL and MySQL (#12512) * Support enums in Postgres table fetch * MySQL support for enum values * null safety * Refactor --- .../src/integrations/microsoftSqlServer.ts | 13 +++--- packages/server/src/integrations/mysql.ts | 18 ++++---- packages/server/src/integrations/oracle.ts | 22 ++++------ packages/server/src/integrations/postgres.ts | 33 ++++++++++---- packages/server/src/integrations/utils.ts | 43 ++++++++++++++++--- 5 files changed, 84 insertions(+), 45 deletions(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 7f25df5ed4..d0a06d4476 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -17,7 +17,7 @@ import { import { getSqlQuery, buildExternalTableId, - convertSqlType, + generateColumnDefinition, finaliseExternalTables, SqlClient, checkExternalTables, @@ -429,15 +429,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { const hasDefault = def.COLUMN_DEFAULT const isAuto = !!autoColumns.find(col => col === name) const required = !!requiredColumns.find(col => col === name) - schema[name] = { + schema[name] = generateColumnDefinition({ autocolumn: isAuto, - name: name, - constraints: { - presence: required && !isAuto && !hasDefault, - }, - ...convertSqlType(def.DATA_TYPE), + name, + presence: required && !isAuto && !hasDefault, externalType: def.DATA_TYPE, - } + }) } tables[tableName] = { _id: buildExternalTableId(datasourceId, tableName), diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index e89393d251..8ec73307f4 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -12,12 +12,13 @@ import { SourceName, Schema, TableSourceType, + FieldType, } from "@budibase/types" import { getSqlQuery, SqlClient, buildExternalTableId, - convertSqlType, + generateColumnDefinition, finaliseExternalTables, checkExternalTables, } from "./utils" @@ -305,16 +306,17 @@ class MySQLIntegration extends Sql implements DatasourcePlus { (column.Extra === "auto_increment" || column.Extra.toLowerCase().includes("generated")) const required = column.Null !== "YES" - const constraints = { - presence: required && !isAuto && !hasDefault, - } - schema[columnName] = { + schema[columnName] = generateColumnDefinition({ name: columnName, autocolumn: isAuto, - constraints, - ...convertSqlType(column.Type), + presence: required && !isAuto && !hasDefault, externalType: column.Type, - } + options: column.Type.startsWith("enum") + ? column.Type.substring(5, column.Type.length - 1) + .split(",") + .map(str => str.replace(/^'(.*)'$/, "$1")) + : undefined, + }) } if (!tables[tableName]) { tables[tableName] = { diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 846aadc242..e9a2dc7998 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -15,7 +15,7 @@ import { import { buildExternalTableId, checkExternalTables, - convertSqlType, + generateColumnDefinition, finaliseExternalTables, getSqlQuery, SqlClient, @@ -250,14 +250,6 @@ class OracleIntegration extends Sql implements DatasourcePlus { ) } - private internalConvertType(column: OracleColumn) { - if (this.isBooleanType(column)) { - return { type: FieldTypes.BOOLEAN } - } - - return convertSqlType(column.type) - } - /** * Fetches the tables from the oracle table and assigns them to the datasource. * @param datasourceId - datasourceId to fetch @@ -302,13 +294,15 @@ class OracleIntegration extends Sql implements DatasourcePlus { const columnName = oracleColumn.name let fieldSchema = table.schema[columnName] if (!fieldSchema) { - fieldSchema = { + fieldSchema = generateColumnDefinition({ autocolumn: OracleIntegration.isAutoColumn(oracleColumn), name: columnName, - constraints: { - presence: false, - }, - ...this.internalConvertType(oracleColumn), + presence: false, + externalType: oracleColumn.type, + }) + + if (this.isBooleanType(oracleColumn)) { + fieldSchema.type = FieldTypes.BOOLEAN } table.schema[columnName] = fieldSchema diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 4d7dc33d75..de3bf0e59e 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -16,7 +16,7 @@ import { import { getSqlQuery, buildExternalTableId, - convertSqlType, + generateColumnDefinition, finaliseExternalTables, SqlClient, checkExternalTables, @@ -162,6 +162,14 @@ class PostgresIntegration extends Sql implements DatasourcePlus { WHERE pg_namespace.nspname = '${this.config.schema}'; ` + ENUM_VALUES = () => ` + SELECT t.typname, + e.enumlabel + FROM pg_type t + JOIN pg_enum e on t.oid = e.enumtypid + JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace; + ` + constructor(config: PostgresConfig) { super(SqlClient.POSTGRES) this.config = config @@ -303,6 +311,18 @@ class PostgresIntegration extends Sql implements DatasourcePlus { const tables: { [key: string]: Table } = {} + // Fetch enum values + const enumsResponse = await this.client.query(this.ENUM_VALUES()) + const enumValues = enumsResponse.rows?.reduce((acc, row) => { + if (!acc[row.typname]) { + return { + [row.typname]: [row.enumlabel], + } + } + acc[row.typname].push(row.enumlabel) + return acc + }, {}) + for (let column of columnsResponse.rows) { const tableName: string = column.table_name const columnName: string = column.column_name @@ -333,16 +353,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus { column.is_generated && column.is_generated !== "NEVER" const isAuto: boolean = hasNextVal || identity || isGenerated const required = column.is_nullable === "NO" - const constraints = { - presence: required && !hasDefault && !isGenerated, - } - tables[tableName].schema[columnName] = { + tables[tableName].schema[columnName] = generateColumnDefinition({ autocolumn: isAuto, name: columnName, - constraints, - ...convertSqlType(column.data_type), + presence: required && !hasDefault && !isGenerated, externalType: column.data_type, - } + options: enumValues?.[column.udt_name], + }) } let finalizedTables = finaliseExternalTables(tables, entities) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index fe8a9055b0..89612cc251 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -67,6 +67,10 @@ const SQL_BOOLEAN_TYPE_MAP = { tinyint: FieldType.BOOLEAN, } +const SQL_OPTIONS_TYPE_MAP = { + "user-defined": FieldType.OPTIONS, +} + const SQL_MISC_TYPE_MAP = { json: FieldType.JSON, bigint: FieldType.BIGINT, @@ -78,6 +82,7 @@ const SQL_TYPE_MAP = { ...SQL_STRING_TYPE_MAP, ...SQL_BOOLEAN_TYPE_MAP, ...SQL_MISC_TYPE_MAP, + ...SQL_OPTIONS_TYPE_MAP, } export enum SqlClient { @@ -178,25 +183,49 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } } -export function convertSqlType(type: string) { +export function generateColumnDefinition(config: { + externalType: string + autocolumn: boolean + name: string + presence: boolean + options?: string[] +}) { + let { externalType, autocolumn, name, presence, options } = config let foundType = FieldType.STRING - const lcType = type.toLowerCase() + const lowerCaseType = externalType.toLowerCase() let matchingTypes = [] for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { - if (lcType.includes(external)) { + if (lowerCaseType.includes(external)) { matchingTypes.push({ external, internal }) } } - //Set the foundType based the longest match + // Set the foundType based the longest match if (matchingTypes.length > 0) { foundType = matchingTypes.reduce((acc, val) => { return acc.external.length >= val.external.length ? acc : val }).internal } - const schema: any = { type: foundType } + + const constraints: { + presence: boolean + inclusion?: string[] + } = { + presence, + } + if (foundType === FieldType.OPTIONS) { + constraints.inclusion = options + } + + const schema: any = { + type: foundType, + externalType, + autocolumn, + name, + constraints, + } if (foundType === FieldType.DATETIME) { - schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType) - schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType) + schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType) + schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType) } return schema }