From 50bbbb2e0621422cc8b59d2fe77d633257ef3f87 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Feb 2024 16:28:00 +0000 Subject: [PATCH 1/3] Updating datasource controller types, this branched out a little bit to removing as many anys as possible, and allowing RowValue to correctly be returned from the allDocs function of the DB. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 3 +- .../backend-core/src/db/instrumentation.ts | 4 +- .../server/src/api/controllers/datasource.ts | 40 +++++++++++++------ .../server/src/api/controllers/integration.ts | 5 ++- .../db/defaultData/datasource_bb_default.ts | 8 ++-- packages/server/src/db/utils.ts | 14 ++++--- packages/server/src/integrations/index.ts | 2 + .../server/src/jsRunner/vm/isolated-vm.ts | 9 ++++- .../server/src/sdk/app/applications/import.ts | 4 +- .../src/sdk/app/datasources/datasources.ts | 2 +- packages/types/src/api/web/app/datasource.ts | 4 +- .../types/src/documents/app/datasource.ts | 17 ++++---- packages/types/src/documents/pouch.ts | 4 +- packages/types/src/sdk/datasources.ts | 1 + packages/types/src/sdk/db.ts | 10 ++++- 15 files changed, 84 insertions(+), 43 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 0e2b4173b0..7e7c997cbe 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -11,6 +11,7 @@ import { Document, isDocument, RowResponse, + RowValue, } from "@budibase/types" import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" @@ -221,7 +222,7 @@ export class DatabaseImpl implements Database { }) } - async allDocs( + async allDocs( params: DatabaseQueryOpts ): Promise> { return this.performCall(db => { diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts index aa2ac424ae..03010d4c92 100644 --- a/packages/backend-core/src/db/instrumentation.ts +++ b/packages/backend-core/src/db/instrumentation.ts @@ -1,5 +1,4 @@ import { - DocumentScope, DocumentDestroyResponse, DocumentInsertResponse, DocumentBulkResponse, @@ -13,6 +12,7 @@ import { DatabasePutOpts, DatabaseQueryOpts, Document, + RowValue, } from "@budibase/types" import tracer from "dd-trace" import { Writable } from "stream" @@ -79,7 +79,7 @@ export class DDInstrumentedDatabase implements Database { }) } - allDocs( + allDocs( params: DatabaseQueryOpts ): Promise> { return tracer.trace("db.allDocs", span => { diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d70c13f800..17217ea018 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -15,10 +15,14 @@ import { FieldType, RelationshipFieldMetadata, SourceName, + UpdateDatasourceRequest, UpdateDatasourceResponse, UserCtx, VerifyDatasourceRequest, VerifyDatasourceResponse, + Table, + RowValue, + DynamicVariable, } from "@budibase/types" import sdk from "../../sdk" import { builderSocket } from "../../websockets" @@ -90,8 +94,10 @@ async function invalidateVariables( existingDatasource: Datasource, updatedDatasource: Datasource ) { - const existingVariables: any = existingDatasource.config?.dynamicVariables - const updatedVariables: any = updatedDatasource.config?.dynamicVariables + const existingVariables: DynamicVariable[] = + existingDatasource.config?.dynamicVariables + const updatedVariables: DynamicVariable[] = + updatedDatasource.config?.dynamicVariables const toInvalidate = [] if (!existingVariables) { @@ -103,9 +109,9 @@ async function invalidateVariables( toInvalidate.push(...existingVariables) } else { // invaldate changed / removed - existingVariables.forEach((existing: any) => { + existingVariables.forEach(existing => { const unchanged = updatedVariables.find( - (updated: any) => + updated => existing.name === updated.name && existing.queryId === updated.queryId && existing.value === updated.value @@ -118,24 +124,32 @@ async function invalidateVariables( await invalidateDynamicVariables(toInvalidate) } -export async function update(ctx: UserCtx) { +export async function update( + ctx: UserCtx +) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId const baseDatasource = await sdk.datasources.get(datasourceId) - const auth = baseDatasource.config?.auth await invalidateVariables(baseDatasource, ctx.request.body) const isBudibaseSource = baseDatasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE - const dataSourceBody = isBudibaseSource - ? { name: ctx.request.body?.name } + const dataSourceBody: Datasource = isBudibaseSource + ? { + name: ctx.request.body?.name, + type: dbCore.BUDIBASE_DATASOURCE_TYPE, + source: SourceName.BUDIBASE, + } : ctx.request.body let datasource: Datasource = { ...baseDatasource, ...sdk.datasources.mergeConfigs(dataSourceBody, baseDatasource), } + + // this block is specific to GSheets, if no auth set, set it back + const auth = baseDatasource.config?.auth if (auth && !ctx.request.body.auth) { // don't strip auth config from DB datasource.config!.auth = auth @@ -204,7 +218,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { const db = context.getAppDB() // Get all internal tables - const internalTables = await db.allDocs( + const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, }) @@ -212,8 +226,8 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { // Filter by datasource and return the docs. const datasourceTableDocs = internalTables.rows.reduce( - (acc: any, table: any) => { - if (table.doc.sourceId == datasourceId) { + (acc: Table[], table) => { + if (table.doc?.sourceId == datasourceId) { acc.push(table.doc) } return acc @@ -254,9 +268,9 @@ export async function destroy(ctx: UserCtx) { if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { await destroyInternalTablesBySourceId(datasourceId) } else { - const queries = await db.allDocs(getQueryParams(datasourceId)) + const queries = await db.allDocs(getQueryParams(datasourceId)) await db.bulkDocs( - queries.rows.map((row: any) => ({ + queries.rows.map(row => ({ _id: row.id, _rev: row.value.rev, _deleted: true, diff --git a/packages/server/src/api/controllers/integration.ts b/packages/server/src/api/controllers/integration.ts index 9cfde31e4c..57038f8401 100644 --- a/packages/server/src/api/controllers/integration.ts +++ b/packages/server/src/api/controllers/integration.ts @@ -1,7 +1,10 @@ import { getDefinition, getDefinitions } from "../../integrations" import { SourceName, UserCtx } from "@budibase/types" -const DISABLED_EXTERNAL_INTEGRATIONS = [SourceName.AIRTABLE] +const DISABLED_EXTERNAL_INTEGRATIONS = [ + SourceName.AIRTABLE, + SourceName.BUDIBASE, +] export async function fetch(ctx: UserCtx) { const definitions = await getDefinitions() diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index ac540cd8fc..03aed3c118 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -1,8 +1,8 @@ import { DEFAULT_BB_DATASOURCE_ID, - DEFAULT_INVENTORY_TABLE_ID, DEFAULT_EMPLOYEE_TABLE_ID, DEFAULT_EXPENSES_TABLE_ID, + DEFAULT_INVENTORY_TABLE_ID, DEFAULT_JOBS_TABLE_ID, } from "../../constants" import { importToRows } from "../../api/controllers/table/utils" @@ -15,19 +15,21 @@ import { expensesImport } from "./expensesImport" import { db as dbCore } from "@budibase/backend-core" import { AutoFieldSubType, + Datasource, FieldType, RelationshipType, Row, + SourceName, Table, TableSchema, TableSourceType, } from "@budibase/types" -const defaultDatasource = { +const defaultDatasource: Datasource = { _id: DEFAULT_BB_DATASOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, name: "Sample Data", - source: "BUDIBASE", + source: SourceName.BUDIBASE, config: {}, } diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 35d9b69e96..983cbf423c 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -1,13 +1,15 @@ import newid from "./newid" import { db as dbCore } from "@budibase/backend-core" import { - FieldType, + DatabaseQueryOpts, + Datasource, DocumentType, FieldSchema, - RelationshipFieldMetadata, - VirtualDocumentType, + FieldType, INTERNAL_TABLE_SOURCE_ID, - DatabaseQueryOpts, + RelationshipFieldMetadata, + SourceName, + VirtualDocumentType, } from "@budibase/types" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -20,11 +22,11 @@ export const enum AppStatus { DEPLOYED = "published", } -export const BudibaseInternalDB = { +export const BudibaseInternalDB: Datasource = { _id: INTERNAL_TABLE_SOURCE_ID, type: dbCore.BUDIBASE_DATASOURCE_TYPE, name: "Budibase DB", - source: "BUDIBASE", + source: SourceName.BUDIBASE, config: {}, } diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 49761bac85..ee2bb23f23 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -37,6 +37,7 @@ const DEFINITIONS: Record = { [SourceName.REDIS]: redis.schema, [SourceName.SNOWFLAKE]: snowflake.schema, [SourceName.ORACLE]: undefined, + [SourceName.BUDIBASE]: undefined, } const INTEGRATIONS: Record = { @@ -56,6 +57,7 @@ const INTEGRATIONS: Record = { [SourceName.REDIS]: redis.integration, [SourceName.SNOWFLAKE]: snowflake.integration, [SourceName.ORACLE]: undefined, + [SourceName.BUDIBASE]: undefined, } // optionally add oracle integration if the oracle binary can be installed diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index b0692f0fd1..928d1b8afa 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -99,7 +99,14 @@ export class IsolatedVM implements VM { } withContext(context: Record, executeWithContext: () => T) { - this.addToContext(context) + this.addToContext({ + ...context, + Snippets: { + specialFunction: function (special: string) { + return "hello world! " + special + }, + }, + }) try { return executeWithContext() diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts index c3415bdb36..f712548fcb 100644 --- a/packages/server/src/sdk/app/applications/import.ts +++ b/packages/server/src/sdk/app/applications/import.ts @@ -85,7 +85,9 @@ async function getImportableDocuments(db: Database) { const docPromises = [] for (let docType of DocumentTypesToImport) { docPromises.push( - db.allDocs(dbCore.getDocParams(docType, null, { include_docs: true })) + db.allDocs( + dbCore.getDocParams(docType, null, { include_docs: true }) + ) ) } // map the responses to the document itself diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index c71c3f1b31..fd0d291d91 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -229,7 +229,7 @@ export async function removeSecretSingle(datasource: Datasource) { } export function mergeConfigs(update: Datasource, old: Datasource) { - if (!update.config) { + if (!update.config || !old.config) { return update } // specific to REST datasources, fix the auth configs again if required diff --git a/packages/types/src/api/web/app/datasource.ts b/packages/types/src/api/web/app/datasource.ts index 4a3d07a952..f931665917 100644 --- a/packages/types/src/api/web/app/datasource.ts +++ b/packages/types/src/api/web/app/datasource.ts @@ -32,9 +32,7 @@ export interface FetchDatasourceInfoResponse { tableNames: string[] } -export interface UpdateDatasourceRequest extends Datasource { - datasource: Datasource -} +export interface UpdateDatasourceRequest extends Datasource {} export interface BuildSchemaFromSourceRequest { tablesFilter?: string[] diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index 67035a2e72..8976e1cae3 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -6,6 +6,9 @@ export interface Datasource extends Document { type: string name?: string source: SourceName + // this is a googlesheets specific property which + // can be found in the GSheets schema - pertains to SSO creds + auth?: { type: string } // the config is defined by the schema config?: Record plus?: boolean @@ -36,6 +39,12 @@ export interface RestAuthConfig { config: RestBasicAuthConfig | RestBearerAuthConfig } +export interface DynamicVariable { + name: string + queryId: string + value: string +} + export interface RestConfig { url: string rejectUnauthorized: boolean @@ -47,11 +56,5 @@ export interface RestConfig { staticVariables: { [key: string]: string } - dynamicVariables: [ - { - name: string - queryId: string - value: string - } - ] + dynamicVariables: DynamicVariable[] } diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index 11efc502be..e1de89649c 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -5,7 +5,7 @@ export interface RowValue { deleted: boolean } -export interface RowResponse { +export interface RowResponse { id: string key: string error: string @@ -13,7 +13,7 @@ export interface RowResponse { doc?: T } -export interface AllDocsResponse { +export interface AllDocsResponse { offset: number total_rows: number rows: RowResponse[] diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 7a335eb3b9..9c0f3a4165 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -56,6 +56,7 @@ export enum SourceName { FIRESTORE = "FIRESTORE", REDIS = "REDIS", SNOWFLAKE = "SNOWFLAKE", + BUDIBASE = "BUDIBASE", } export enum IncludeRelationship { diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 9e44a4827f..c4e4a4f02f 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,5 +1,11 @@ import type Nano from "@budibase/nano" -import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../" +import { + AllDocsResponse, + AnyDocument, + Document, + RowValue, + ViewTemplateOpts, +} from "../" import { Writable } from "stream" export enum SearchIndex { @@ -135,7 +141,7 @@ export interface Database { opts?: DatabasePutOpts ): Promise bulkDocs(documents: AnyDocument[]): Promise - allDocs( + allDocs( params: DatabaseQueryOpts ): Promise> query( From 92ac417fdf8beb167fa4e29cc38630df48be415f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Feb 2024 16:31:50 +0000 Subject: [PATCH 2/3] Removing accidentally commited stuff. --- packages/server/src/jsRunner/vm/isolated-vm.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 928d1b8afa..b0692f0fd1 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -99,14 +99,7 @@ export class IsolatedVM implements VM { } withContext(context: Record, executeWithContext: () => T) { - this.addToContext({ - ...context, - Snippets: { - specialFunction: function (special: string) { - return "hello world! " + special - }, - }, - }) + this.addToContext(context) try { return executeWithContext() From a91ecb8c1443c9d2d91d4c0e97418bf73de281cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Feb 2024 17:03:04 +0000 Subject: [PATCH 3/3] Addressing PR comments. --- packages/server/src/api/controllers/datasource.ts | 4 ++-- packages/types/src/documents/pouch.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 17217ea018..0f17c5a2f5 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -95,9 +95,9 @@ async function invalidateVariables( updatedDatasource: Datasource ) { const existingVariables: DynamicVariable[] = - existingDatasource.config?.dynamicVariables + existingDatasource.config?.dynamicVariables || [] const updatedVariables: DynamicVariable[] = - updatedDatasource.config?.dynamicVariables + updatedDatasource.config?.dynamicVariables || [] const toInvalidate = [] if (!existingVariables) { diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index e1de89649c..6ff851a515 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -9,7 +9,7 @@ export interface RowResponse { id: string key: string error: string - value: T | RowValue + value: T doc?: T }