Making integrations more like what custom integrations look like (to simplify integration).

packages/server/src/api/controllers/integration.js

@@ -1,12 +1,11 @@
-const { cloneDeep } = require("lodash")
+const { getDefinitions } = require("../../integrations")
-const { definitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = cloneDeep(definitions)
+  const defs = await getDefinitions()
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
+  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = definitions[ctx.params.type]
+  ctx.body = defs[ctx.params.type]
 }

packages/server/src/api/controllers/plugin.ts

@@ -3,6 +3,22 @@ import { extractPluginTarball } from "../../utilities/fileSystem"
 import { getGlobalDB } from "@budibase/backend-core/tenancy"
 import { generatePluginID, getPluginParams } from "../../db/utils"
 import { uploadDirectory } from "@budibase/backend-core/objectStore"
+import { PluginType } from "@budibase/types"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
 
 export async function upload(ctx: any) {
   const plugins =
@@ -68,13 +84,7 @@ export async function upload(ctx: any) {
 }
 
 export async function fetch(ctx: any) {
-  const db = getGlobalDB()
+  ctx.body = await getPlugins()
-  const response = await db.allDocs(
-    getPluginParams(null, {
-      include_docs: true,
-    })
-  )
-  ctx.body = response.rows.map((row: any) => row.doc)
 }
 
 export async function destroy(ctx: any) {}

packages/server/src/integrations/airtable.ts

@@ -5,7 +5,6 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-module AirtableModule {
 const Airtable = require("airtable")
 
 interface AirtableConfig {
@@ -143,8 +142,7 @@ module AirtableModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: AirtableIntegration,
 }
-}

packages/server/src/integrations/arangodb.ts

@@ -5,7 +5,6 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-module ArangoModule {
 const { Database, aql } = require("arangojs")
 
 interface ArangodbConfig {
@@ -103,8 +102,7 @@ module ArangoModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: ArangoDBIntegration,
 }
-}

packages/server/src/integrations/couchdb.ts

@@ -5,7 +5,6 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-module CouchDBModule {
 const PouchDB = require("pouchdb")
 
 interface CouchDBConfig {
@@ -95,19 +94,14 @@ module CouchDBModule {
   }
 
   async delete(query: { id: string }) {
-      const doc = await this.query(
+    const doc = await this.query("get", "Cannot find doc to be deleted", query)
-        "get",
-        "Cannot find doc to be deleted",
-        query
-      )
     return this.query("remove", "Error deleting couchDB document", {
       json: doc,
     })
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: CouchDBIntegration,
 }
-}

packages/server/src/integrations/dynamodb.ts

@@ -5,7 +5,6 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-module DynamoModule {
 const AWS = require("aws-sdk")
 const { AWS_REGION } = require("../db/dynamoClient")
 
@@ -225,8 +224,7 @@ module DynamoModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: DynamoDBIntegration,
 }
-}

packages/server/src/integrations/elasticsearch.ts

@@ -5,7 +5,6 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-module ElasticsearchModule {
 const { Client } = require("@elastic/elasticsearch")
 
 interface ElasticsearchConfig {
@@ -148,8 +147,7 @@ module ElasticsearchModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: ElasticSearchIntegration,
 }
-}

packages/server/src/integrations/firebase.ts

@@ -6,7 +6,6 @@ import {
 } from "@budibase/types"
 import { Firestore, WhereFilterOp } from "@google-cloud/firestore"
 
-module Firebase {
 interface FirebaseConfig {
   email: string
   privateKey: string
@@ -182,8 +181,7 @@ module Firebase {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: FirebaseIntegration,
 }
-}

packages/server/src/integrations/googlesheets.ts

@@ -13,7 +13,6 @@ import { DataSourceOperation, FieldTypes } from "../constants"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 import env from "../environment"
 
-module GoogleSheetsModule {
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { getScopedConfig } = require("@budibase/backend-core/db")
 const { Configs } = require("@budibase/backend-core/constants")
@@ -152,9 +151,7 @@ module GoogleSheetsModule {
   async fetchAccessToken(
     payload: AuthTokenRequest
   ): Promise<AuthTokenResponse> {
-      const response = await fetch(
+    const response = await fetch("https://www.googleapis.com/oauth2/v4/token", {
-        "https://www.googleapis.com/oauth2/v4/token",
-        {
       method: "POST",
       body: JSON.stringify({
         ...payload,
@@ -163,8 +160,7 @@ module GoogleSheetsModule {
       headers: {
         "Content-Type": "application/json",
       },
-        }
+    })
-      )
 
     const json = await response.json()
 
@@ -385,11 +381,7 @@ module GoogleSheetsModule {
         }
         await row.save()
         return [
-            this.buildRowObject(
+          this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber),
-              sheet.headerValues,
-              row._rawData,
-              row._rowNumber
-            ),
         ]
       } else {
         throw new Error("Row does not exist.")
@@ -414,8 +406,7 @@ module GoogleSheetsModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: GoogleSheetsIntegration,
 }
-}

packages/server/src/integrations/index.ts

@@ -1,22 +1,24 @@
-const postgres = require("./postgres")
+import postgres from "./postgres"
-const dynamodb = require("./dynamodb")
+import dynamodb from "./dynamodb"
-const mongodb = require("./mongodb")
+import mongodb from "./mongodb"
-const elasticsearch = require("./elasticsearch")
+import elasticsearch from "./elasticsearch"
-const couchdb = require("./couchdb")
+import couchdb from "./couchdb"
-const sqlServer = require("./microsoftSqlServer")
+import sqlServer from "./microsoftSqlServer"
-const s3 = require("./s3")
+import s3 from "./s3"
-const airtable = require("./airtable")
+import airtable from "./airtable"
-const mysql = require("./mysql")
+import mysql from "./mysql"
-const arangodb = require("./arangodb")
+import arangodb from "./arangodb"
-const rest = require("./rest")
+import rest from "./rest"
-const googlesheets = require("./googlesheets")
+import googlesheets from "./googlesheets"
-const firebase = require("./firebase")
+import firebase from "./firebase"
-const redis = require("./redis")
+import redis from "./redis"
-const snowflake = require("./snowflake")
+import snowflake from "./snowflake"
-const { SourceName } = require("@budibase/types")
+import { getPlugins } from "../api/controllers/plugin"
+import { SourceName, Integration, PluginType } from "@budibase/types"
 const environment = require("../environment")
+const { cloneDeep } = require("lodash")
 
-const DEFINITIONS = {
+const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
   [SourceName.DYNAMODB]: dynamodb.schema,
   [SourceName.MONGODB]: mongodb.schema,
@@ -33,7 +35,7 @@ const DEFINITIONS = {
   [SourceName.SNOWFLAKE]: snowflake.schema,
 }
 
-const INTEGRATIONS = {
+const INTEGRATIONS: { [key: string]: any } = {
   [SourceName.POSTGRES]: postgres.integration,
   [SourceName.DYNAMODB]: dynamodb.integration,
   [SourceName.MONGODB]: mongodb.integration,
@@ -48,7 +50,7 @@ const INTEGRATIONS = {
   [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.GOOGLE_SHEETS]: googlesheets.integration,
   [SourceName.REDIS]: redis.integration,
-  [SourceName.FIREBASE]: firebase.integration,
+  [SourceName.FIRESTORE]: firebase.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
 }
 
@@ -64,6 +66,9 @@ if (environment.SELF_HOSTED) {
 }
 
 module.exports = {
-  definitions: DEFINITIONS,
+  getDefinitions: async () => {
+    const custom = await getPlugins(PluginType.DATASOURCE)
+    return cloneDeep(DEFINITIONS)
+  },
   integrations: INTEGRATIONS,
 }

packages/server/src/integrations/microsoftSqlServer.ts

@@ -18,7 +18,6 @@ import {
 } from "./utils"
 import Sql from "./base/sql"
 
-module MSSQLModule {
 const sqlServer = require("mssql")
 const DEFAULT_SCHEMA = "dbo"
 
@@ -302,8 +301,7 @@ module MSSQLModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: SqlServerIntegration,
 }
-}

packages/server/src/integrations/mongodb.ts

@@ -15,7 +15,6 @@ import {
   CommonOptions,
 } from "mongodb"
 
-module MongoDBModule {
 interface MongoDBConfig {
   connectionString: string
   db: string
@@ -96,9 +95,7 @@ module MongoDBModule {
           (field === "_id" || field?.startsWith("$")) &&
           typeof json[field] === "string"
         ) {
-            const id = json[field].match(
+          const id = json[field].match(/(?<=objectid\(['"]).*(?=['"]\))/gi)?.[0]
-              /(?<=objectid\(['"]).*(?=['"]\))/gi
-            )?.[0]
           if (id) {
             json[field] = ObjectID.createFromHexString(id)
           }
@@ -320,8 +317,7 @@ module MongoDBModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: MongoIntegration,
 }
-}

packages/server/src/integrations/mysql.ts

@@ -19,7 +19,6 @@ import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
 import Sql from "./base/sql"
 
-module MySQLModule {
 const mysql = require("mysql2/promise")
 
 interface MySQLConfig {
@@ -216,10 +215,7 @@ module MySQLModule {
         )
         for (let column of descResp) {
           const columnName = column.Field
-            if (
+          if (column.Key === "PRI" && primaryKeys.indexOf(column.Key) === -1) {
-              column.Key === "PRI" &&
-              primaryKeys.indexOf(column.Key) === -1
-            ) {
             primaryKeys.push(columnName)
           }
           const constraints = {
@@ -285,8 +281,7 @@ module MySQLModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: MySQLIntegration,
 }
-}

packages/server/src/integrations/oracle.ts

@@ -25,7 +25,6 @@ import oracledb, {
 import Sql from "./base/sql"
 import { FieldTypes } from "../constants"
 
-module OracleModule {
 oracledb.outFormat = oracledb.OUT_FORMAT_OBJECT
 
 interface OracleConfig {
@@ -268,10 +267,7 @@ module OracleModule {
           const condition = c.searchCondition
             .replace(/\s/g, "") // remove spaces
             .replace(/[']+/g, "") // remove quotes
-            if (
+          if (condition.includes("in(0,1)") || condition.includes("in(1,0)")) {
-              condition.includes("in(0,1)") ||
-              condition.includes("in(1,0)")
-            ) {
             return true
           }
         }
@@ -335,13 +331,11 @@ module OracleModule {
           }
 
           // iterate each constraint on the column
-            Object.values(oracleColumn.constraints).forEach(
+          Object.values(oracleColumn.constraints).forEach(oracleConstraint => {
-              oracleConstraint => {
             if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
               table.primary!.push(columnName)
             }
-              }
+          })
-            )
         })
     })
 
@@ -454,8 +448,7 @@ module OracleModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: OracleIntegration,
 }
-}

packages/server/src/integrations/postgres.ts

@@ -16,7 +16,6 @@ import {
 } from "./utils"
 import Sql from "./base/sql"
 
-module PostgresModule {
 const { Client, types } = require("pg")
 const { escapeDangerousCharacters } = require("../utilities")
 
@@ -221,9 +220,7 @@ module PostgresModule {
     let tableKeys: { [key: string]: string[] } = {}
     await this.openConnection()
     try {
-        const primaryKeysResponse = await this.client.query(
+      const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
-          this.PRIMARY_KEYS_SQL
-        )
       for (let table of primaryKeysResponse.rows) {
         const tableName = table.table_name
         if (!tableKeys[tableName]) {
@@ -325,8 +322,7 @@ module PostgresModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: PostgresIntegration,
 }
-}

packages/server/src/integrations/redis.ts

@@ -1,7 +1,6 @@
 import { DatasourceFieldType, Integration, QueryType } from "@budibase/types"
 import Redis from "ioredis"
 
-module RedisModule {
 interface RedisConfig {
   host: string
   port: number
@@ -142,8 +141,7 @@ module RedisModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: RedisIntegration,
 }
-}

packages/server/src/integrations/rest.ts

@@ -14,6 +14,11 @@ import {
   BearerAuthConfig,
 } from "../definitions/datasource"
 import { get } from "lodash"
+const fetch = require("node-fetch")
+const { formatBytes } = require("../utilities")
+const { performance } = require("perf_hooks")
+const FormData = require("form-data")
+const { URLSearchParams } = require("url")
 
 const BodyTypes = {
   NONE: "none",
@@ -50,16 +55,7 @@ const coreFields = {
   },
 }
 
-module RestModule {
+const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
-  const fetch = require("node-fetch")
-  const { formatBytes } = require("../utilities")
-  const { performance } = require("perf_hooks")
-  const FormData = require("form-data")
-  const { URLSearchParams } = require("url")
-  const {
-    parseStringPromise: xmlParser,
-    Builder: XmlBuilder,
-  } = require("xml2js")
 
 const SCHEMA: Integration = {
   docs: "https://github.com/node-fetch/node-fetch",
@@ -404,9 +400,8 @@ module RestModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: RestIntegration,
   AuthType,
 }
-}

packages/server/src/integrations/s3.ts

@@ -1,6 +1,4 @@
 import { Integration, QueryType, IntegrationBase } from "@budibase/types"
-
-module S3Module {
 const AWS = require("aws-sdk")
 
 interface S3Config {
@@ -79,8 +77,7 @@ module S3Module {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: S3Integration,
 }
-}

packages/server/src/integrations/snowflake.ts

@@ -1,7 +1,6 @@
 import { Integration, QueryType, SqlQuery } from "@budibase/types"
 import { Snowflake } from "snowflake-promise"
 
-module SnowflakeModule {
 interface SnowflakeConfig {
   account: string
   username: string
@@ -92,8 +91,7 @@ module SnowflakeModule {
   }
 }
 
-  module.exports = {
+export default {
   schema: SCHEMA,
   integration: SnowflakeIntegration,
 }
-}

packages/types/src/documents/index.ts

@@ -1,5 +1,6 @@
 export * from "./account"
 export * from "./app"
 export * from "./global"
+export * from "./plugin"
 export * from "./platform"
 export * from "./document"

packages/types/src/documents/plugin/index.ts

@@ -0,0 +1,4 @@
+export enum PluginType {
+  DATASOURCE = "datasource",
+  COMPONENT = "component",
+}