Merge pull request #15106 from Budibase/fix-options-with-commas-and-types

Fix MySQL options column imports that have commas or other type names in them.

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -169,7 +169,9 @@ const descriptions = datasourceDescribe({
 })
 
 if (descriptions.length) {
-  describe.each(descriptions)("$dbName", ({ config, dsProvider }) => {
+  describe.each(descriptions)(
+    "$dbName",
+    ({ config, dsProvider, isOracle, isMSSQL }) => {
       let datasource: Datasource
       let rawDatasource: Datasource
       let client: Knex
@@ -209,7 +211,9 @@ if (descriptions.length) {
       describe("list", () => {
         it("returns all the datasources", async () => {
           const datasources = await config.api.datasource.fetch()
-        expect(datasources).toContainEqual(expect.objectContaining(datasource))
+          expect(datasources).toContainEqual(
+            expect.objectContaining(datasource)
+          )
         })
       })
 
@@ -310,7 +314,7 @@ if (descriptions.length) {
                 presence: {
                   allowEmpty: false,
                 },
-              inclusion: [],
+                inclusion: ["1", "2", "3"],
               },
             },
             [FieldType.NUMBER]: {
@@ -412,6 +416,92 @@ if (descriptions.length) {
           }
           expect(updated).toEqual(expected)
         })
+
+        !isOracle &&
+          !isMSSQL &&
+          it("can fetch options columns with a large number of options", async () => {
+            const enumOptions = new Array(1000)
+              .fill(0)
+              .map((_, i) => i.toString())
+              .toSorted()
+            await client.schema.createTable("options", table => {
+              table.increments("id").primary()
+              table.enum("enum", enumOptions, {
+                useNative: true,
+                enumName: "enum",
+              })
+            })
+
+            const resp = await config.api.datasource.fetchSchema({
+              datasourceId: datasource._id!,
+            })
+            expect(resp.errors).toEqual({})
+
+            const table = resp.datasource.entities!.options
+            expect(
+              table.schema.enum.constraints!.inclusion!.toSorted()
+            ).toEqual(enumOptions)
+          })
+
+        !isOracle &&
+          !isMSSQL &&
+          it("can fetch options with commas in them", async () => {
+            const enumOptions = [
+              "Lincoln, Abraham",
+              "Washington, George",
+              "Fred",
+              "Bob",
+            ].toSorted()
+            await client.schema.createTable("options", table => {
+              table.increments("id").primary()
+              table.enum("enum", enumOptions, {
+                useNative: true,
+                enumName: "enum",
+              })
+            })
+
+            const resp = await config.api.datasource.fetchSchema({
+              datasourceId: datasource._id!,
+            })
+            expect(resp.errors).toEqual({})
+
+            const table = resp.datasource.entities!.options
+            expect(
+              table.schema.enum.constraints!.inclusion!.toSorted()
+            ).toEqual(enumOptions)
+          })
+
+        !isOracle &&
+          !isMSSQL &&
+          it("can fetch options that may include other type names", async () => {
+            const enumOptions = [
+              "int",
+              "bigint",
+              "float",
+              "numeric",
+              "json",
+              "map",
+            ].toSorted()
+
+            await client.schema.createTable("options", table => {
+              table.increments("id").primary()
+              table.enum("enum", enumOptions, {
+                useNative: true,
+                enumName: "enum",
+              })
+            })
+
+            const resp = await config.api.datasource.fetchSchema({
+              datasourceId: datasource._id!,
+            })
+
+            expect(resp.errors).toEqual({})
+
+            const table = resp.datasource.entities!.options
+            expect(
+              table.schema.enum.constraints!.inclusion!.toSorted()
+            ).toEqual(enumOptions)
+          })
       })
 
       describe("verify", () => {
@@ -495,5 +585,6 @@ if (descriptions.length) {
           )
         })
       })
-  })
+    }
+  )
 }

packages/server/src/integrations/mysql.ts

@@ -322,9 +322,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
             presence: required && !isAuto && !hasDefault,
             externalType: column.Type,
             options: column.Type.startsWith("enum")
-              ? column.Type.substring(5, column.Type.length - 1)
-                  .split(",")
-                  .map(str => str.replace(/^'(.*)'$/, "$1"))
+              ? column.Type.substring(6, column.Type.length - 2).split("','")
               : undefined,
           })
         }

packages/server/src/integrations/utils/utils.ts

@@ -138,12 +138,22 @@ export function generateColumnDefinition(config: {
   let { externalType, autocolumn, name, presence, options } = config
   let foundType = FieldType.STRING
   const lowerCaseType = externalType.toLowerCase()
-  let matchingTypes = []
+  let matchingTypes: { external: string; internal: PrimitiveTypes }[] = []
+
+  // In at least MySQL, the external type of an ENUM column is "enum('option1',
+  // 'option2', ...)", which can potentially contain any type name as a
+  // substring. To get around this interfering with the loop below, we first
+  // check for an enum column and handle that separately.
+  if (lowerCaseType.startsWith("enum")) {
+    matchingTypes.push({ external: "enum", internal: FieldType.OPTIONS })
+  } else {
     for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
       if (lowerCaseType.includes(external)) {
         matchingTypes.push({ external, internal })
       }
     }
+  }
+
   // Set the foundType based the longest match
   if (matchingTypes.length > 0) {
     foundType = matchingTypes.reduce((acc, val) => {