Adding custom datasources to UI.

packages/builder/src/components/backend/DatasourceNavigator/icons/Custom.svelte

@@ -0,0 +1,39 @@
+<script>
+  export let width = "100"
+  export let height = "100"
+  let color =
+    "var(--spectrum-heading-xxs-text-color, var(--spectrum-alias-heading-text-color))"
+</script>
+
+<svg
+  {width}
+  {height}
+  xmlns="http://www.w3.org/2000/svg"
+  x="0px"
+  y="0px"
+  viewBox="0 0 230.795 230.795"
+  style="enable-background:new 0 0 230.795 230.795;"
+  xml:space="preserve"
+>
+  <g>
+    <path
+      d="M60.357,63.289c-2.929-2.929-7.678-2.93-10.606-0.001L2.197,110.836C0.79,112.243,0,114.151,0,116.14
+		c0,1.989,0.79,3.896,2.196,5.303l47.348,47.35c1.465,1.465,3.384,2.197,5.304,2.197c1.919,0,3.839-0.732,5.303-2.196
+		c2.93-2.929,2.93-7.678,0.001-10.606L18.107,116.14l42.25-42.245C63.286,70.966,63.286,66.217,60.357,63.289z"
+      fill={color}
+    />
+    <path
+      d="M228.598,110.639l-47.355-47.352c-2.928-2.928-7.677-2.929-10.606,0.001c-2.929,2.929-2.929,7.678,0.001,10.607
+		l42.051,42.048l-42.249,42.243c-2.93,2.929-2.93,7.678-0.001,10.606c1.465,1.465,3.384,2.197,5.304,2.197
+		c1.919,0,3.839-0.732,5.303-2.196l47.554-47.547c1.407-1.406,2.197-3.314,2.197-5.304
+		C230.795,113.954,230.005,112.046,228.598,110.639z"
+      fill={color}
+    />
+    <path
+      d="M155.889,61.302c-3.314-2.484-8.017-1.806-10.498,1.51l-71.994,96.184c-2.482,3.316-1.807,8.017,1.51,10.498
+		c1.348,1.01,2.925,1.496,4.488,1.496c2.282,0,4.537-1.038,6.01-3.006L157.398,71.8C159.881,68.484,159.205,63.784,155.889,61.302z"
+      fill={color}
+    />
+  </g>
+  <g />
+</svg>

packages/builder/src/components/backend/DatasourceNavigator/icons/index.js

@@ -15,6 +15,7 @@ import GoogleSheets from "./GoogleSheets.svelte"
 import Firebase from "./Firebase.svelte"
 import Redis from "./Redis.svelte"
 import Snowflake from "./Snowflake.svelte"
+import Custom from "./Custom.svelte"
 
 export default {
   BUDIBASE: Budibase,
@@ -34,4 +35,5 @@ export default {
   FIRESTORE: Firebase,
   REDIS: Redis,
   SNOWFLAKE: Snowflake,
+  CUSTOM: Custom,
 }

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -92,6 +92,14 @@
     }
     integrations = newIntegrations
   }
+
+  function getIcon(integrationType, schema) {
+    if (schema.custom) {
+      return ICONS.CUSTOM
+    } else {
+      return ICONS[integrationType]
+    }
+  }
 </script>
 
 <Modal bind:this={internalTableModal}>
@@ -158,7 +166,7 @@
           >
             <div class="item-body" class:with-type={!!schema.type}>
               <svelte:component
-                this={ICONS[integrationType]}
+                this={getIcon(integrationType, schema)}
                 height="20"
                 width="20"
               />

packages/server/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = integrations[datasource.source]
+  const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)

packages/server/src/integrations/base/query.ts

@@ -1,11 +1,11 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { integrations } = require("../index")
+const { getIntegration } = require("../index")
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
-  const Integration = integrations[datasource.source]
+  const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {
     const integration = new Integration(datasource.config)

packages/server/src/integrations/index.ts

@@ -67,8 +67,22 @@ if (environment.SELF_HOSTED) {
 
 module.exports = {
   getDefinitions: async () => {
-    const custom = await getPlugins(PluginType.DATASOURCE)
-    return cloneDeep(DEFINITIONS)
-  },
-  integrations: INTEGRATIONS,
+    const plugins = await getPlugins(PluginType.DATASOURCE)
+    // extract the actual schema from each custom
+    const pluginSchemas: { [key: string]: Integration } = {}
+    for (let plugin of plugins) {
+      const sourceId = plugin.name
+      pluginSchemas[sourceId] = {
+        ...plugin.schema["schema"],
+        custom: true,
+      }
+    }
+    return {
+      ...cloneDeep(DEFINITIONS),
+      ...pluginSchemas,
+    }
+  },
+  getIntegration: async () => {
+    return INTEGRATIONS
+  },
 }

packages/server/src/threads/query.ts

@@ -2,7 +2,7 @@ import { default as threadUtils } from "./utils"
 threadUtils.threadSetup()
 import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
 const ScriptRunner = require("../utilities/scriptRunner")
-const { integrations } = require("../integrations")
+const { getIntegration } = require("../integrations")
 const { processStringSync } = require("@budibase/string-templates")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 const {
@@ -62,7 +62,7 @@ class QueryRunner {
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
-    const Integration = integrations[datasourceClone.source]
+    const Integration = await getIntegration(datasourceClone.source)
     if (!Integration) {
       throw "Integration type does not exist."
     }