Merge pull request #12732 from Budibase/chore/move-code-to-sdk

Move datasource code to sdk

packages/server/src/api/controllers/datasource.ts

@@ -1,13 +1,10 @@
-import {
+import { getQueryParams, getTableParams } from "../../db/utils"
-  DocumentType,
-  generateDatasourceID,
-  getQueryParams,
-  getTableParams,
-} from "../../db/utils"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { context, db as dbCore, events } from "@budibase/backend-core"
 import {
+  BuildSchemaFromSourceRequest,
+  BuildSchemaFromSourceResponse,
   CreateDatasourceRequest,
   CreateDatasourceResponse,
   Datasource,
@@ -22,7 +19,6 @@ import {
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
 export async function fetch(ctx: UserCtx) {
@@ -67,22 +63,16 @@ export async function information(
   }
 }
 
-export async function buildSchemaFromDb(ctx: UserCtx) {
+export async function buildSchemaFromSource(
-  const db = context.getAppDB()
+  ctx: UserCtx<BuildSchemaFromSourceRequest, BuildSchemaFromSourceResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
   const tablesFilter = ctx.request.body.tablesFilter
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+  const { datasource, errors } = await sdk.datasources.buildSchemaFromSource(
-    datasource,
+    datasourceId,
     tablesFilter
   )
-  datasource.entities = tables
-
-  setDefaultDisplayColumns(datasource)
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
-  )
-  datasource._rev = dbResp.rev
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
@@ -90,24 +80,6 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   }
 }
 
-/**
- * Make sure all datasource entities have a display name selected
- */
-function setDefaultDisplayColumns(datasource: Datasource) {
-  //
-  for (let entity of Object.values(datasource.entities || {})) {
-    if (entity.primaryDisplay) {
-      continue
-    }
-    const notAutoColumn = Object.values(entity.schema).find(
-      schema => !schema.autocolumn
-    )
-    if (notAutoColumn) {
-      entity.primaryDisplay = notAutoColumn.name
-    }
-  }
-}
-
 /**
  * Check for variables that have been updated or removed and invalidate them.
  */
@@ -205,54 +177,18 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   }
 }
 
-const preSaveAction: Partial<Record<SourceName, any>> = {
-  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
-    await googleSetupCreationAuth(datasource.config as any)
-  },
-}
-
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
-  const db = context.getAppDB()
+  const {
-  const plus = ctx.request.body.datasource.plus
+    datasource: datasourceData,
-  const fetchSchema = ctx.request.body.fetchSchema
+    fetchSchema,
-  const tablesFilter = ctx.request.body.tablesFilter
+    tablesFilter,
-
+  } = ctx.request.body
-  const datasource = {
+  const { datasource, errors } = await sdk.datasources.save(datasourceData, {
-    _id: generateDatasourceID({ plus }),
+    fetchSchema,
-    ...ctx.request.body.datasource,
+    tablesFilter,
-    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  })
-  }
-
-  let errors: Record<string, string> = {}
-  if (fetchSchema) {
-    const schema = await sdk.datasources.buildFilteredSchema(
-      datasource,
-      tablesFilter
-    )
-    datasource.entities = schema.tables
-    setDefaultDisplayColumns(datasource)
-    errors = schema.errors
-  }
-
-  if (preSaveAction[datasource.source]) {
-    await preSaveAction[datasource.source](datasource)
-  }
-
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
-  )
-  await events.datasource.created(datasource)
-  datasource._rev = dbResp.rev
-
-  // Drain connection pools when configuration is changed
-  if (datasource.source) {
-    const source = await getIntegration(datasource.source)
-    if (source && source.pool) {
-      await source.pool.end()
-    }
-  }
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),

packages/server/src/api/routes/datasource.ts

@@ -53,7 +53,7 @@ router
   .post(
     "/api/datasources/:datasourceId/schema",
     authorized(permissions.BUILDER),
-    datasourceController.buildSchemaFromDb
+    datasourceController.buildSchemaFromSource
   )
   .post(
     "/api/datasources",

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,4 +1,4 @@
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events } from "@budibase/backend-core"
 import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
@@ -14,16 +14,22 @@ import {
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
-import { getDefinitions, getDefinition } from "../../../integrations"
+import {
+  getDefinitions,
+  getDefinition,
+  getIntegration,
+} from "../../../integrations"
 import merge from "lodash/merge"
 import {
   BudibaseInternalDB,
+  generateDatasourceID,
   getDatasourceParams,
   getDatasourcePlusParams,
   getTableParams,
+  DocumentType,
 } from "../../../db/utils"
 import sdk from "../../index"
-import datasource from "../../../api/routes/datasource"
+import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets"
 
 const ENV_VAR_PREFIX = "env."
 
@@ -273,3 +279,75 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
 
   return externalDatasources.rows.map(r => r.doc!)
 }
+
+export async function save(
+  datasource: Datasource,
+  opts?: { fetchSchema?: boolean; tablesFilter?: string[] }
+): Promise<{ datasource: Datasource; errors: Record<string, string> }> {
+  const db = context.getAppDB()
+  const plus = datasource.plus
+
+  const fetchSchema = opts?.fetchSchema || false
+  const tablesFilter = opts?.tablesFilter || []
+
+  datasource = {
+    _id: generateDatasourceID({ plus }),
+    ...datasource,
+    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  }
+
+  let errors: Record<string, string> = {}
+  if (fetchSchema) {
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
+    datasource.entities = schema.tables
+    setDefaultDisplayColumns(datasource)
+    errors = schema.errors
+  }
+
+  if (preSaveAction[datasource.source]) {
+    await preSaveAction[datasource.source](datasource)
+  }
+
+  const dbResp = await db.put(
+    sdk.tables.populateExternalTableSchemas(datasource)
+  )
+  await events.datasource.created(datasource)
+  datasource._rev = dbResp.rev
+
+  // Drain connection pools when configuration is changed
+  if (datasource.source) {
+    const source = await getIntegration(datasource.source)
+    if (source && source.pool) {
+      await source.pool.end()
+    }
+  }
+
+  return { datasource, errors }
+}
+
+const preSaveAction: Partial<Record<SourceName, any>> = {
+  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
+    await googleSetupCreationAuth(datasource.config as any)
+  },
+}
+
+/**
+ * Make sure all datasource entities have a display name selected
+ */
+export function setDefaultDisplayColumns(datasource: Datasource) {
+  //
+  for (let entity of Object.values(datasource.entities || {})) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+}

packages/server/src/sdk/app/datasources/plus.ts

@@ -5,7 +5,9 @@ import {
   Schema,
 } from "@budibase/types"
 import * as datasources from "./datasources"
+import tableSdk from "../tables"
 import { getIntegration } from "../../../integrations"
+import { context } from "@budibase/backend-core"
 
 export async function buildFilteredSchema(
   datasource: Datasource,
@@ -60,3 +62,24 @@ export async function getAndMergeDatasource(datasource: Datasource) {
   }
   return await datasources.enrich(datasource)
 }
+
+export async function buildSchemaFromSource(
+  datasourceId: string,
+  tablesFilter?: string[]
+) {
+  const db = context.getAppDB()
+
+  const datasource = await datasources.get(datasourceId)
+
+  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
+
+  datasources.setDefaultDisplayColumns(datasource)
+  const dbResp = await db.put(tableSdk.populateExternalTableSchemas(datasource))
+  datasource._rev = dbResp.rev
+
+  return {
+    datasource,
+    errors,
+  }
+}

packages/types/src/api/web/app/datasource.ts

@@ -35,3 +35,12 @@ export interface FetchDatasourceInfoResponse {
 export interface UpdateDatasourceRequest extends Datasource {
   datasource: Datasource
 }
+
+export interface BuildSchemaFromSourceRequest {
+  tablesFilter?: string[]
+}
+
+export interface BuildSchemaFromSourceResponse {
+  datasource: Datasource
+  errors: Record<string, string>
+}