Merge branch 'fix/massive-attachment-export' of github.com:Budibase/budibase into fix/massive-attachment-export

charts/budibase/templates/proxy-service-deployment.yaml

@@ -16,6 +16,7 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: budibase-proxy
+  minReadySeconds: 10
   strategy:
     type: RollingUpdate
   template:

packages/server/src/integrations/microsoftSqlServer.ts

@@ -17,7 +17,7 @@ import {
 import {
   getSqlQuery,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   SqlClient,
   checkExternalTables,
@@ -429,15 +429,12 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         const hasDefault = def.COLUMN_DEFAULT
         const isAuto = !!autoColumns.find(col => col === name)
         const required = !!requiredColumns.find(col => col === name)
-        schema[name] = {
+        schema[name] = generateColumnDefinition({
           autocolumn: isAuto,
-          name: name,
+          name,
-          constraints: {
           presence: required && !isAuto && !hasDefault,
-          },
-          ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,
-        }
+        })
       }
       tables[tableName] = {
         _id: buildExternalTableId(datasourceId, tableName),

packages/server/src/integrations/mysql.ts

@@ -12,12 +12,13 @@ import {
   SourceName,
   Schema,
   TableSourceType,
+  FieldType,
 } from "@budibase/types"
 import {
   getSqlQuery,
   SqlClient,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   checkExternalTables,
 } from "./utils"
@@ -305,16 +306,17 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
             (column.Extra === "auto_increment" ||
               column.Extra.toLowerCase().includes("generated"))
           const required = column.Null !== "YES"
-          const constraints = {
+          schema[columnName] = generateColumnDefinition({
-            presence: required && !isAuto && !hasDefault,
-          }
-          schema[columnName] = {
             name: columnName,
             autocolumn: isAuto,
-            constraints,
+            presence: required && !isAuto && !hasDefault,
-            ...convertSqlType(column.Type),
             externalType: column.Type,
-          }
+            options: column.Type.startsWith("enum")
+              ? column.Type.substring(5, column.Type.length - 1)
+                  .split(",")
+                  .map(str => str.replace(/^'(.*)'$/, "$1"))
+              : undefined,
+          })
         }
         if (!tables[tableName]) {
           tables[tableName] = {

packages/server/src/integrations/oracle.ts

@@ -15,7 +15,7 @@ import {
 import {
   buildExternalTableId,
   checkExternalTables,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   getSqlQuery,
   SqlClient,
@@ -250,14 +250,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
     )
   }
 
-  private internalConvertType(column: OracleColumn) {
-    if (this.isBooleanType(column)) {
-      return { type: FieldTypes.BOOLEAN }
-    }
-
-    return convertSqlType(column.type)
-  }
-
   /**
    * Fetches the tables from the oracle table and assigns them to the datasource.
    * @param datasourceId - datasourceId to fetch
@@ -302,13 +294,15 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           const columnName = oracleColumn.name
           let fieldSchema = table.schema[columnName]
           if (!fieldSchema) {
-            fieldSchema = {
+            fieldSchema = generateColumnDefinition({
               autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
               name: columnName,
-              constraints: {
               presence: false,
-              },
+              externalType: oracleColumn.type,
-              ...this.internalConvertType(oracleColumn),
+            })
+
+            if (this.isBooleanType(oracleColumn)) {
+              fieldSchema.type = FieldTypes.BOOLEAN
             }
 
             table.schema[columnName] = fieldSchema

packages/server/src/integrations/postgres.ts

@@ -16,7 +16,7 @@ import {
 import {
   getSqlQuery,
   buildExternalTableId,
-  convertSqlType,
+  generateColumnDefinition,
   finaliseExternalTables,
   SqlClient,
   checkExternalTables,
@@ -162,6 +162,14 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   WHERE pg_namespace.nspname = '${this.config.schema}';
   `
 
+  ENUM_VALUES = () => `
+  SELECT t.typname,  
+      e.enumlabel
+  FROM pg_type t 
+  JOIN pg_enum e on t.oid = e.enumtypid  
+  JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace;
+  `
+
   constructor(config: PostgresConfig) {
     super(SqlClient.POSTGRES)
     this.config = config
@@ -303,6 +311,18 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
       const tables: { [key: string]: Table } = {}
 
+      // Fetch enum values
+      const enumsResponse = await this.client.query(this.ENUM_VALUES())
+      const enumValues = enumsResponse.rows?.reduce((acc, row) => {
+        if (!acc[row.typname]) {
+          return {
+            [row.typname]: [row.enumlabel],
+          }
+        }
+        acc[row.typname].push(row.enumlabel)
+        return acc
+      }, {})
+
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name
         const columnName: string = column.column_name
@@ -333,16 +353,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.is_generated && column.is_generated !== "NEVER"
         const isAuto: boolean = hasNextVal || identity || isGenerated
         const required = column.is_nullable === "NO"
-        const constraints = {
+        tables[tableName].schema[columnName] = generateColumnDefinition({
-          presence: required && !hasDefault && !isGenerated,
-        }
-        tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
-          constraints,
+          presence: required && !hasDefault && !isGenerated,
-          ...convertSqlType(column.data_type),
           externalType: column.data_type,
-        }
+          options: enumValues?.[column.udt_name],
+        })
       }
 
       let finalizedTables = finaliseExternalTables(tables, entities)

packages/server/src/integrations/utils.ts

@@ -67,6 +67,10 @@ const SQL_BOOLEAN_TYPE_MAP = {
   tinyint: FieldType.BOOLEAN,
 }
 
+const SQL_OPTIONS_TYPE_MAP = {
+  "user-defined": FieldType.OPTIONS,
+}
+
 const SQL_MISC_TYPE_MAP = {
   json: FieldType.JSON,
   bigint: FieldType.BIGINT,
@@ -78,6 +82,7 @@ const SQL_TYPE_MAP = {
   ...SQL_STRING_TYPE_MAP,
   ...SQL_BOOLEAN_TYPE_MAP,
   ...SQL_MISC_TYPE_MAP,
+  ...SQL_OPTIONS_TYPE_MAP,
 }
 
 export enum SqlClient {
@@ -178,25 +183,49 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
   }
 }
 
-export function convertSqlType(type: string) {
+export function generateColumnDefinition(config: {
+  externalType: string
+  autocolumn: boolean
+  name: string
+  presence: boolean
+  options?: string[]
+}) {
+  let { externalType, autocolumn, name, presence, options } = config
   let foundType = FieldType.STRING
-  const lcType = type.toLowerCase()
+  const lowerCaseType = externalType.toLowerCase()
   let matchingTypes = []
   for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
-    if (lcType.includes(external)) {
+    if (lowerCaseType.includes(external)) {
       matchingTypes.push({ external, internal })
     }
   }
-  //Set the foundType based the longest match
+  // Set the foundType based the longest match
   if (matchingTypes.length > 0) {
     foundType = matchingTypes.reduce((acc, val) => {
       return acc.external.length >= val.external.length ? acc : val
     }).internal
   }
-  const schema: any = { type: foundType }
+
+  const constraints: {
+    presence: boolean
+    inclusion?: string[]
+  } = {
+    presence,
+  }
+  if (foundType === FieldType.OPTIONS) {
+    constraints.inclusion = options
+  }
+
+  const schema: any = {
+    type: foundType,
+    externalType,
+    autocolumn,
+    name,
+    constraints,
+  }
   if (foundType === FieldType.DATETIME) {
-    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
+    schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)
-    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
+    schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType)
   }
   return schema
 }