From 92eb185390ef52b84cbdfe591f4536f72cf57eff Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 8 Jan 2024 18:54:57 +0100 Subject: [PATCH 1/3] Type BuildSchemaFromSource --- packages/server/src/api/controllers/datasource.ts | 6 +++++- packages/types/src/api/web/app/datasource.ts | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 8c177f1704..76cc63bfb7 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -8,6 +8,8 @@ import { getIntegration } from "../../integrations" import { invalidateDynamicVariables } from "../../threads/utils" import { context, db as dbCore, events } from "@budibase/backend-core" import { + BuildSchemaFromSourceRequest, + BuildSchemaFromSourceResponse, CreateDatasourceRequest, CreateDatasourceResponse, Datasource, @@ -67,7 +69,9 @@ export async function information( } } -export async function buildSchemaFromDb(ctx: UserCtx) { +export async function buildSchemaFromDb( + ctx: UserCtx +) { const db = context.getAppDB() const tablesFilter = ctx.request.body.tablesFilter const datasource = await sdk.datasources.get(ctx.params.datasourceId) diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index 9cd3c8f4bb..4a3d07a952 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -35,3 +35,12 @@ export interface FetchDatasourceInfoResponse { export interface UpdateDatasourceRequest extends Datasource { datasource: Datasource } + +export interface BuildSchemaFromSourceRequest { + tablesFilter?: string[] +} + +export interface BuildSchemaFromSourceResponse { + datasource: Datasource + errors: Record +} From 8e038e61b0a0ee6cddc5456eaa63903b563ca3d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 8 Jan 2024 19:02:39 +0100 Subject: [PATCH 2/3] Move datasource.save to sdk --- .../server/src/api/controllers/datasource.ts | 62 +++----------- .../src/sdk/app/datasources/datasources.ts | 84 ++++++++++++++++++- 2 files changed, 91 insertions(+), 55 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 76cc63bfb7..6961011b0d 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -1,9 +1,4 @@ -import { - DocumentType, - generateDatasourceID, - getQueryParams, - getTableParams, -} from "../../db/utils" +import { getQueryParams, getTableParams } from "../../db/utils" import { getIntegration } from "../../integrations" import { invalidateDynamicVariables } from "../../threads/utils" import { context, db as dbCore, events } from "@budibase/backend-core" @@ -24,7 +19,6 @@ import { } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" -import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" import { isEqual } from "lodash" export async function fetch(ctx: UserCtx) { @@ -209,54 +203,18 @@ export async function update(ctx: UserCtx) { } } -const preSaveAction: Partial> = { - [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { - await googleSetupCreationAuth(datasource.config as any) - }, -} - export async function save( ctx: UserCtx ) { - const db = context.getAppDB() - const plus = ctx.request.body.datasource.plus - const fetchSchema = ctx.request.body.fetchSchema - const tablesFilter = ctx.request.body.tablesFilter - - const datasource = { - _id: generateDatasourceID({ plus }), - ...ctx.request.body.datasource, - type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE, - } - - let errors: Record = {} - if (fetchSchema) { - const schema = await sdk.datasources.buildFilteredSchema( - datasource, - tablesFilter - ) - datasource.entities = schema.tables - setDefaultDisplayColumns(datasource) - errors = schema.errors - } - - if (preSaveAction[datasource.source]) { - await preSaveAction[datasource.source](datasource) - } - - const dbResp = await db.put( - sdk.tables.populateExternalTableSchemas(datasource) - ) - await events.datasource.created(datasource) - datasource._rev = dbResp.rev - - // Drain connection pools when configuration is changed - if (datasource.source) { - const source = await getIntegration(datasource.source) - if (source && source.pool) { - await source.pool.end() - } - } + const { + datasource: datasourceData, + fetchSchema, + tablesFilter, + } = ctx.request.body + const { datasource, errors } = await sdk.datasources.save(datasourceData, { + fetchSchema, + tablesFilter, + }) ctx.body = { datasource: await sdk.datasources.removeSecretSingle(datasource), diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index 51cceeab94..bb0d8bcb13 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -1,4 +1,4 @@ -import { context, db as dbCore } from "@budibase/backend-core" +import { context, db as dbCore, events } from "@budibase/backend-core" import { findHBSBlocks, processObjectSync } from "@budibase/string-templates" import { Datasource, @@ -14,16 +14,22 @@ import { } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { getEnvironmentVariables } from "../../utils" -import { getDefinitions, getDefinition } from "../../../integrations" +import { + getDefinitions, + getDefinition, + getIntegration, +} from "../../../integrations" import merge from "lodash/merge" import { BudibaseInternalDB, + generateDatasourceID, getDatasourceParams, getDatasourcePlusParams, getTableParams, + DocumentType, } from "../../../db/utils" import sdk from "../../index" -import datasource from "../../../api/routes/datasource" +import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets" const ENV_VAR_PREFIX = "env." @@ -273,3 +279,75 @@ export async function getExternalDatasources(): Promise { return externalDatasources.rows.map(r => r.doc!) } + +export async function save( + datasource: Datasource, + opts?: { fetchSchema?: boolean; tablesFilter?: string[] } +): Promise<{ datasource: Datasource; errors: Record }> { + const db = context.getAppDB() + const plus = datasource.plus + + const fetchSchema = opts?.fetchSchema || false + const tablesFilter = opts?.tablesFilter || [] + + datasource = { + _id: generateDatasourceID({ plus }), + ...datasource, + type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE, + } + + let errors: Record = {} + if (fetchSchema) { + const schema = await sdk.datasources.buildFilteredSchema( + datasource, + tablesFilter + ) + datasource.entities = schema.tables + setDefaultDisplayColumns(datasource) + errors = schema.errors + } + + if (preSaveAction[datasource.source]) { + await preSaveAction[datasource.source](datasource) + } + + const dbResp = await db.put( + sdk.tables.populateExternalTableSchemas(datasource) + ) + await events.datasource.created(datasource) + datasource._rev = dbResp.rev + + // Drain connection pools when configuration is changed + if (datasource.source) { + const source = await getIntegration(datasource.source) + if (source && source.pool) { + await source.pool.end() + } + } + + return { datasource, errors } +} + +const preSaveAction: Partial> = { + [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => { + await googleSetupCreationAuth(datasource.config as any) + }, +} + +/** + * Make sure all datasource entities have a display name selected + */ +function setDefaultDisplayColumns(datasource: Datasource) { + // + for (let entity of Object.values(datasource.entities || {})) { + if (entity.primaryDisplay) { + continue + } + const notAutoColumn = Object.values(entity.schema).find( + schema => !schema.autocolumn + ) + if (notAutoColumn) { + entity.primaryDisplay = notAutoColumn.name + } + } +} From 3f6e1bbaa21798ed6635cf83cce5e543363c0bbe Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 8 Jan 2024 19:07:47 +0100 Subject: [PATCH 3/3] Move datasource.buildSchemaFromDb to sdk --- .../server/src/api/controllers/datasource.ts | 34 +++---------------- packages/server/src/api/routes/datasource.ts | 2 +- .../src/sdk/app/datasources/datasources.ts | 2 +- .../server/src/sdk/app/datasources/plus.ts | 23 +++++++++++++ 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 6961011b0d..a702a8cd84 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -63,24 +63,16 @@ export async function information( } } -export async function buildSchemaFromDb( +export async function buildSchemaFromSource( ctx: UserCtx ) { - const db = context.getAppDB() + const datasourceId = ctx.params.datasourceId const tablesFilter = ctx.request.body.tablesFilter - const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const { tables, errors } = await sdk.datasources.buildFilteredSchema( - datasource, + const { datasource, errors } = await sdk.datasources.buildSchemaFromSource( + datasourceId, tablesFilter ) - datasource.entities = tables - - setDefaultDisplayColumns(datasource) - const dbResp = await db.put( - sdk.tables.populateExternalTableSchemas(datasource) - ) - datasource._rev = dbResp.rev ctx.body = { datasource: await sdk.datasources.removeSecretSingle(datasource), @@ -88,24 +80,6 @@ export async function buildSchemaFromDb( } } -/** - * Make sure all datasource entities have a display name selected - */ -function setDefaultDisplayColumns(datasource: Datasource) { - // - for (let entity of Object.values(datasource.entities || {})) { - if (entity.primaryDisplay) { - continue - } - const notAutoColumn = Object.values(entity.schema).find( - schema => !schema.autocolumn - ) - if (notAutoColumn) { - entity.primaryDisplay = notAutoColumn.name - } - } -} - /** * Check for variables that have been updated or removed and invalidate them. */ diff --git a/packages/server/src/api/routes/datasource.ts b/packages/server/src/api/routes/datasource.ts index 7b4945806a..755088c56c 100644 --- a/packages/server/src/api/routes/datasource.ts +++ b/packages/server/src/api/routes/datasource.ts @@ -53,7 +53,7 @@ router .post( "/api/datasources/:datasourceId/schema", authorized(permissions.BUILDER), - datasourceController.buildSchemaFromDb + datasourceController.buildSchemaFromSource ) .post( "/api/datasources", diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index bb0d8bcb13..c71c3f1b31 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -337,7 +337,7 @@ const preSaveAction: Partial> = { /** * Make sure all datasource entities have a display name selected */ -function setDefaultDisplayColumns(datasource: Datasource) { +export function setDefaultDisplayColumns(datasource: Datasource) { // for (let entity of Object.values(datasource.entities || {})) { if (entity.primaryDisplay) { diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts index 117d19a6a7..04cd508863 100644 --- a/packages/server/src/sdk/app/datasources/plus.ts +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -5,7 +5,9 @@ import { Schema, } from "@budibase/types" import * as datasources from "./datasources" +import tableSdk from "../tables" import { getIntegration } from "../../../integrations" +import { context } from "@budibase/backend-core" export async function buildFilteredSchema( datasource: Datasource, @@ -60,3 +62,24 @@ export async function getAndMergeDatasource(datasource: Datasource) { } return await datasources.enrich(datasource) } + +export async function buildSchemaFromSource( + datasourceId: string, + tablesFilter?: string[] +) { + const db = context.getAppDB() + + const datasource = await datasources.get(datasourceId) + + const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter) + datasource.entities = tables + + datasources.setDefaultDisplayColumns(datasource) + const dbResp = await db.put(tableSdk.populateExternalTableSchemas(datasource)) + datasource._rev = dbResp.rev + + return { + datasource, + errors, + } +}