Merge pull request #12012 from Budibase/fix/external-table-keep-types

Fix fetching external tables with user column type

.github/workflows/budibase_ci.yml

@@ -270,18 +270,23 @@ jobs:
 
           if [[ $branch == "master" ]]; then
             base_commit=$(git rev-parse origin/master)
-          else
+          elif [[ $branch == "develop" ]]; then
             base_commit=$(git rev-parse origin/develop)
           fi
 
-          echo "target_branch=$branch"
+          if [[ ! -z $base_commit ]]; then
-          echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
+            echo "target_branch=$branch"
-          echo "pro_commit=$pro_commit"
+            echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
-          echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
+            echo "pro_commit=$pro_commit"
-          echo "base_commit=$base_commit"
+            echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
-          echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
+            echo "base_commit=$base_commit"
+            echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT"
+          else
+            echo "Nothing to do - branch to branch merge."
+          fi
 
-      - name: Check submodule merged to develop
+      - name: Check submodule merged to base branch
+        if: ${{ steps.get_pro_commits.outputs.base_commit != '' }}
         uses: actions/github-script@v4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -290,7 +295,7 @@ jobs:
             const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
 
             if (submoduleCommit !== baseCommit) {
-              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
+              console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.');
               console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
               process.exit(1);
             } else {

packages/server/src/integrations/utils.ts

@@ -1,7 +1,12 @@
-import { SqlQuery, Table, SearchFilters, Datasource } from "@budibase/types"
+import {
+  SqlQuery,
+  Table,
+  SearchFilters,
+  Datasource,
+  FieldType,
+} from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
 import {
-  FieldTypes,
   BuildSchemaErrors,
   InvalidColumns,
   NoEmptyFilterStrings,
@@ -13,57 +18,57 @@ const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
 
 const SQL_NUMBER_TYPE_MAP = {
-  integer: FieldTypes.NUMBER,
+  integer: FieldType.NUMBER,
-  int: FieldTypes.NUMBER,
+  int: FieldType.NUMBER,
-  decimal: FieldTypes.NUMBER,
+  decimal: FieldType.NUMBER,
-  smallint: FieldTypes.NUMBER,
+  smallint: FieldType.NUMBER,
-  real: FieldTypes.NUMBER,
+  real: FieldType.NUMBER,
-  float: FieldTypes.NUMBER,
+  float: FieldType.NUMBER,
-  numeric: FieldTypes.NUMBER,
+  numeric: FieldType.NUMBER,
-  mediumint: FieldTypes.NUMBER,
+  mediumint: FieldType.NUMBER,
-  dec: FieldTypes.NUMBER,
+  dec: FieldType.NUMBER,
-  double: FieldTypes.NUMBER,
+  double: FieldType.NUMBER,
-  fixed: FieldTypes.NUMBER,
+  fixed: FieldType.NUMBER,
-  "double precision": FieldTypes.NUMBER,
+  "double precision": FieldType.NUMBER,
-  number: FieldTypes.NUMBER,
+  number: FieldType.NUMBER,
-  binary_float: FieldTypes.NUMBER,
+  binary_float: FieldType.NUMBER,
-  binary_double: FieldTypes.NUMBER,
+  binary_double: FieldType.NUMBER,
-  money: FieldTypes.NUMBER,
+  money: FieldType.NUMBER,
-  smallmoney: FieldTypes.NUMBER,
+  smallmoney: FieldType.NUMBER,
 }
 
 const SQL_DATE_TYPE_MAP = {
-  timestamp: FieldTypes.DATETIME,
+  timestamp: FieldType.DATETIME,
-  time: FieldTypes.DATETIME,
+  time: FieldType.DATETIME,
-  datetime: FieldTypes.DATETIME,
+  datetime: FieldType.DATETIME,
-  smalldatetime: FieldTypes.DATETIME,
+  smalldatetime: FieldType.DATETIME,
-  date: FieldTypes.DATETIME,
+  date: FieldType.DATETIME,
 }
 
 const SQL_DATE_ONLY_TYPES = ["date"]
 const SQL_TIME_ONLY_TYPES = ["time"]
 
 const SQL_STRING_TYPE_MAP = {
-  varchar: FieldTypes.STRING,
+  varchar: FieldType.STRING,
-  char: FieldTypes.STRING,
+  char: FieldType.STRING,
-  nchar: FieldTypes.STRING,
+  nchar: FieldType.STRING,
-  nvarchar: FieldTypes.STRING,
+  nvarchar: FieldType.STRING,
-  ntext: FieldTypes.STRING,
+  ntext: FieldType.STRING,
-  enum: FieldTypes.STRING,
+  enum: FieldType.STRING,
-  blob: FieldTypes.STRING,
+  blob: FieldType.STRING,
-  long: FieldTypes.STRING,
+  long: FieldType.STRING,
-  text: FieldTypes.STRING,
+  text: FieldType.STRING,
 }
 
 const SQL_BOOLEAN_TYPE_MAP = {
-  boolean: FieldTypes.BOOLEAN,
+  boolean: FieldType.BOOLEAN,
-  bit: FieldTypes.BOOLEAN,
+  bit: FieldType.BOOLEAN,
-  tinyint: FieldTypes.BOOLEAN,
+  tinyint: FieldType.BOOLEAN,
 }
 
 const SQL_MISC_TYPE_MAP = {
-  json: FieldTypes.JSON,
+  json: FieldType.JSON,
-  bigint: FieldTypes.BIGINT,
+  bigint: FieldType.BIGINT,
 }
 
 const SQL_TYPE_MAP = {
@@ -154,7 +159,7 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
 }
 
 export function convertSqlType(type: string) {
-  let foundType = FieldTypes.STRING
+  let foundType = FieldType.STRING
   const lcType = type.toLowerCase()
   let matchingTypes = []
   for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) {
@@ -169,7 +174,7 @@ export function convertSqlType(type: string) {
     }).internal
   }
   const schema: any = { type: foundType }
-  if (foundType === FieldTypes.DATETIME) {
+  if (foundType === FieldType.DATETIME) {
     schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType)
     schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType)
   }
@@ -212,7 +217,7 @@ export function shouldCopyRelationship(
   tableIds: string[]
 ) {
   return (
-    column.type === FieldTypes.LINK &&
+    column.type === FieldType.LINK &&
     column.tableId &&
     tableIds.includes(column.tableId)
   )
@@ -230,22 +235,23 @@ export function shouldCopySpecialColumn(
   column: { type: string },
   fetchedColumn: { type: string } | undefined
 ) {
-  const isFormula = column.type === FieldTypes.FORMULA
+  const isFormula = column.type === FieldType.FORMULA
   const specialTypes = [
-    FieldTypes.OPTIONS,
+    FieldType.OPTIONS,
-    FieldTypes.LONGFORM,
+    FieldType.LONGFORM,
-    FieldTypes.ARRAY,
+    FieldType.ARRAY,
-    FieldTypes.FORMULA,
+    FieldType.FORMULA,
+    FieldType.BB_REFERENCE,
   ]
   // column has been deleted, remove - formulas will never exist, always copy
   if (!isFormula && column && !fetchedColumn) {
     return false
   }
   const fetchedIsNumber =
-    !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
+    !fetchedColumn || fetchedColumn.type === FieldType.NUMBER
   return (
-    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
+    specialTypes.indexOf(column.type as FieldType) !== -1 ||
-    (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
+    (fetchedIsNumber && column.type === FieldType.BOOLEAN)
   )
 }