From 6df4bf84df1d166af6c2ce206d891b764cdf1aa2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Jan 2024 15:42:16 +0100 Subject: [PATCH 1/4] Lint --- packages/server/src/api/controllers/datasource.ts | 7 ++----- packages/server/src/integrations/postgres.ts | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 0a7430aa94..b794312fb6 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -57,17 +57,14 @@ async function getAndMergeDatasource(datasource: Datasource) { async function buildSchemaHelper(datasource: Datasource): Promise { const connector = (await getConnector(datasource)) as DatasourcePlus - return await connector.buildSchema( - datasource._id!, - datasource.entities! as Record - ) + return await connector.buildSchema(datasource._id!, datasource.entities!) } async function buildFilteredSchema( datasource: Datasource, filter?: string[] ): Promise { - let schema = await buildSchemaHelper(datasource) + const schema = await buildSchemaHelper(datasource) if (!filter) { return schema } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index de3bf0e59e..2d61d05a01 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -362,8 +362,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus { }) } - let finalizedTables = finaliseExternalTables(tables, entities) - let errors = checkExternalTables(finalizedTables) + const finalizedTables = finaliseExternalTables(tables, entities) + const errors = checkExternalTables(finalizedTables) return { tables: finalizedTables, errors } } catch (err) { // @ts-ignore From dd77779bc16fb1e7794afbec89497ac0d4745f2b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 5 Jan 2024 15:14:31 +0100 Subject: [PATCH 2/4] Move code to sdk --- .../server/src/api/controllers/datasource.ts | 80 +++---------------- .../server/src/sdk/app/datasources/index.ts | 2 + .../server/src/sdk/app/datasources/plus.ts | 71 ++++++++++++++++ 3 files changed, 86 insertions(+), 67 deletions(-) create mode 100644 packages/server/src/sdk/app/datasources/plus.ts diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index b794312fb6..364490e5f5 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -14,10 +14,7 @@ import { DatasourcePlus, FetchDatasourceInfoRequest, FetchDatasourceInfoResponse, - IntegrationBase, - Schema, SourceName, - Table, UpdateDatasourceResponse, UserCtx, VerifyDatasourceRequest, @@ -28,62 +25,6 @@ import { builderSocket } from "../../websockets" import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets" import { isEqual } from "lodash" -async function getConnector( - datasource: Datasource -): Promise { - const Connector = await getIntegration(datasource.source) - // can't enrich if it doesn't have an ID yet - if (datasource._id) { - datasource = await sdk.datasources.enrich(datasource) - } - // Connect to the DB and build the schema - return new Connector(datasource.config) -} - -async function getAndMergeDatasource(datasource: Datasource) { - let existingDatasource: undefined | Datasource - if (datasource._id) { - existingDatasource = await sdk.datasources.get(datasource._id) - } - let enrichedDatasource = datasource - if (existingDatasource) { - enrichedDatasource = sdk.datasources.mergeConfigs( - datasource, - existingDatasource - ) - } - return await sdk.datasources.enrich(enrichedDatasource) -} - -async function buildSchemaHelper(datasource: Datasource): Promise { - const connector = (await getConnector(datasource)) as DatasourcePlus - return await connector.buildSchema(datasource._id!, datasource.entities!) -} - -async function buildFilteredSchema( - datasource: Datasource, - filter?: string[] -): Promise { - const schema = await buildSchemaHelper(datasource) - if (!filter) { - return schema - } - - let filteredSchema: Schema = { tables: {}, errors: {} } - for (let key in schema.tables) { - if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { - filteredSchema.tables[key] = schema.tables[key] - } - } - - for (let key in schema.errors) { - if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { - filteredSchema.errors[key] = schema.errors[key] - } - } - return filteredSchema -} - export async function fetch(ctx: UserCtx) { ctx.body = await sdk.datasources.fetch() } @@ -92,8 +33,7 @@ export async function verify( ctx: UserCtx ) { const { datasource } = ctx.request.body - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = await getConnector(enrichedDatasource) + const connector = await sdk.datasources.getConnector(datasource) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } @@ -109,8 +49,9 @@ export async function information( ctx: UserCtx ) { const { datasource } = ctx.request.body - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus + const connector = (await sdk.datasources.getConnector( + datasource + )) as DatasourcePlus if (!connector.getTableNames) { ctx.throw(400, "Table name fetching not supported by datasource") } @@ -125,7 +66,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) { const tablesFilter = ctx.request.body.tablesFilter const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter) + const { tables, errors } = await sdk.datasources.buildFilteredSchema( + datasource, + tablesFilter + ) datasource.entities = tables setDefaultDisplayColumns(datasource) @@ -277,7 +221,10 @@ export async function save( let errors: Record = {} if (fetchSchema) { - const schema = await buildFilteredSchema(datasource, tablesFilter) + const schema = await sdk.datasources.buildFilteredSchema( + datasource, + tablesFilter + ) datasource.entities = schema.tables setDefaultDisplayColumns(datasource) errors = schema.errors @@ -381,8 +328,7 @@ export async function query(ctx: UserCtx) { export async function getExternalSchema(ctx: UserCtx) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const enrichedDatasource = await getAndMergeDatasource(datasource) - const connector = await getConnector(enrichedDatasource) + const connector = await sdk.datasources.getConnector(datasource) if (!connector.getExternalSchema) { ctx.throw(400, "Datasource does not support exporting external schema") diff --git a/packages/server/src/sdk/app/datasources/index.ts b/packages/server/src/sdk/app/datasources/index.ts index 1ce6b0e689..8f06e989d3 100644 --- a/packages/server/src/sdk/app/datasources/index.ts +++ b/packages/server/src/sdk/app/datasources/index.ts @@ -1,5 +1,7 @@ import * as datasources from "./datasources" +import * as plus from "./plus" export default { ...datasources, + ...plus, } diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts new file mode 100644 index 0000000000..65b908526c --- /dev/null +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -0,0 +1,71 @@ +import { + Datasource, + DatasourcePlus, + IntegrationBase, + Schema, +} from "@budibase/types" +import * as datasources from "./datasources" +import { getIntegration } from "../../../integrations" + +export async function buildFilteredSchema( + datasource: Datasource, + filter?: string[] +): Promise { + const schema = await buildSchemaHelper(datasource) + if (!filter) { + return schema + } + + let filteredSchema: Schema = { tables: {}, errors: {} } + for (let key in schema.tables) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.tables[key] = schema.tables[key] + } + } + + for (let key in schema.errors) { + if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) { + filteredSchema.errors[key] = schema.errors[key] + } + } + return filteredSchema +} + +async function buildSchemaHelper(datasource: Datasource): Promise { + const connector = (await getConnector(datasource)) as DatasourcePlus + const externalSchema = await connector.buildSchema( + datasource._id!, + datasource.entities! + ) + return externalSchema +} + +export async function getConnector( + datasource: Datasource +): Promise { + const Connector = await getIntegration(datasource.source) + + datasource = await getAndMergeDatasource(datasource) + + // can't enrich if it doesn't have an ID yet + if (datasource._id) { + datasource = await datasources.enrich(datasource) + } + // Connect to the DB and build the schema + return new Connector(datasource.config) +} + +async function getAndMergeDatasource(datasource: Datasource) { + let existingDatasource: undefined | Datasource + if (datasource._id) { + existingDatasource = await datasources.get(datasource._id) + } + let enrichedDatasource = datasource + if (existingDatasource) { + enrichedDatasource = datasources.mergeConfigs( + datasource, + existingDatasource + ) + } + return await datasources.enrich(enrichedDatasource) +} From 1f9ee5a2066e18331fcd5967b5f32b5e2ecfc37e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 5 Jan 2024 16:27:45 +0100 Subject: [PATCH 3/4] Clean code --- .../server/src/sdk/app/datasources/plus.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts index 65b908526c..0049730119 100644 --- a/packages/server/src/sdk/app/datasources/plus.ts +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -45,7 +45,7 @@ export async function getConnector( ): Promise { const Connector = await getIntegration(datasource.source) - datasource = await getAndMergeDatasource(datasource) + datasource = await mergeAndEnrich(datasource) // can't enrich if it doesn't have an ID yet if (datasource._id) { @@ -55,17 +55,11 @@ export async function getConnector( return new Connector(datasource.config) } -async function getAndMergeDatasource(datasource: Datasource) { - let existingDatasource: undefined | Datasource +async function mergeAndEnrich(datasource: Datasource) { if (datasource._id) { - existingDatasource = await datasources.get(datasource._id) + const existingDatasource = await datasources.get(datasource._id) + + datasource = datasources.mergeConfigs(datasource, existingDatasource) } - let enrichedDatasource = datasource - if (existingDatasource) { - enrichedDatasource = datasources.mergeConfigs( - datasource, - existingDatasource - ) - } - return await datasources.enrich(enrichedDatasource) + return await datasources.enrich(datasource) } From 498137ba7c8a1a40d6382f2f81abc0ddeac43af5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 5 Jan 2024 16:39:55 +0100 Subject: [PATCH 4/4] Split back --- packages/server/src/api/controllers/datasource.ts | 15 ++++++++++++--- packages/server/src/sdk/app/datasources/plus.ts | 5 +---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 364490e5f5..8c177f1704 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -33,7 +33,10 @@ export async function verify( ctx: UserCtx ) { const { datasource } = ctx.request.body - const connector = await sdk.datasources.getConnector(datasource) + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( + datasource + ) + const connector = await sdk.datasources.getConnector(enrichedDatasource) if (!connector.testConnection) { ctx.throw(400, "Connection information verification not supported") } @@ -49,8 +52,11 @@ export async function information( ctx: UserCtx ) { const { datasource } = ctx.request.body - const connector = (await sdk.datasources.getConnector( + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( datasource + ) + const connector = (await sdk.datasources.getConnector( + enrichedDatasource )) as DatasourcePlus if (!connector.getTableNames) { ctx.throw(400, "Table name fetching not supported by datasource") @@ -328,7 +334,10 @@ export async function query(ctx: UserCtx) { export async function getExternalSchema(ctx: UserCtx) { const datasource = await sdk.datasources.get(ctx.params.datasourceId) - const connector = await sdk.datasources.getConnector(datasource) + const enrichedDatasource = await sdk.datasources.getAndMergeDatasource( + datasource + ) + const connector = await sdk.datasources.getConnector(enrichedDatasource) if (!connector.getExternalSchema) { ctx.throw(400, "Datasource does not support exporting external schema") diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts index 0049730119..117d19a6a7 100644 --- a/packages/server/src/sdk/app/datasources/plus.ts +++ b/packages/server/src/sdk/app/datasources/plus.ts @@ -44,9 +44,6 @@ export async function getConnector( datasource: Datasource ): Promise { const Connector = await getIntegration(datasource.source) - - datasource = await mergeAndEnrich(datasource) - // can't enrich if it doesn't have an ID yet if (datasource._id) { datasource = await datasources.enrich(datasource) @@ -55,7 +52,7 @@ export async function getConnector( return new Connector(datasource.config) } -async function mergeAndEnrich(datasource: Datasource) { +export async function getAndMergeDatasource(datasource: Datasource) { if (datasource._id) { const existingDatasource = await datasources.get(datasource._id)