Merge pull request #13166 from Budibase/types/update-datasource-controller

Updating datasource controller typing

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -11,6 +11,7 @@ import {
   Document,
   isDocument,
   RowResponse,
+  RowValue,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
@@ -221,7 +222,7 @@ export class DatabaseImpl implements Database {
     })
   }
 
-  async allDocs<T extends Document>(
+  async allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
     return this.performCall(db => {

packages/backend-core/src/db/instrumentation.ts

@@ -1,5 +1,4 @@
 import {
-  DocumentScope,
   DocumentDestroyResponse,
   DocumentInsertResponse,
   DocumentBulkResponse,
@@ -13,6 +12,7 @@ import {
   DatabasePutOpts,
   DatabaseQueryOpts,
   Document,
+  RowValue,
 } from "@budibase/types"
 import tracer from "dd-trace"
 import { Writable } from "stream"
@@ -79,7 +79,7 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
-  allDocs<T extends Document>(
+  allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
     return tracer.trace("db.allDocs", span => {

packages/server/src/api/controllers/datasource.ts

@@ -15,10 +15,14 @@ import {
   FieldType,
   RelationshipFieldMetadata,
   SourceName,
+  UpdateDatasourceRequest,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  Table,
+  RowValue,
+  DynamicVariable,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
@@ -90,8 +94,10 @@ async function invalidateVariables(
   existingDatasource: Datasource,
   updatedDatasource: Datasource
 ) {
-  const existingVariables: any = existingDatasource.config?.dynamicVariables
-  const updatedVariables: any = updatedDatasource.config?.dynamicVariables
+  const existingVariables: DynamicVariable[] =
+    existingDatasource.config?.dynamicVariables || []
+  const updatedVariables: DynamicVariable[] =
+    updatedDatasource.config?.dynamicVariables || []
   const toInvalidate = []
 
   if (!existingVariables) {
@@ -103,9 +109,9 @@ async function invalidateVariables(
     toInvalidate.push(...existingVariables)
   } else {
     // invaldate changed / removed
-    existingVariables.forEach((existing: any) => {
+    existingVariables.forEach(existing => {
       const unchanged = updatedVariables.find(
-        (updated: any) =>
+        updated =>
           existing.name === updated.name &&
           existing.queryId === updated.queryId &&
           existing.value === updated.value
@@ -118,24 +124,32 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
+export async function update(
+  ctx: UserCtx<UpdateDatasourceRequest, UpdateDatasourceResponse>
+) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
   const baseDatasource = await sdk.datasources.get(datasourceId)
-  const auth = baseDatasource.config?.auth
   await invalidateVariables(baseDatasource, ctx.request.body)
 
   const isBudibaseSource =
     baseDatasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
 
-  const dataSourceBody = isBudibaseSource
-    ? { name: ctx.request.body?.name }
+  const dataSourceBody: Datasource = isBudibaseSource
+    ? {
+        name: ctx.request.body?.name,
+        type: dbCore.BUDIBASE_DATASOURCE_TYPE,
+        source: SourceName.BUDIBASE,
+      }
     : ctx.request.body
 
   let datasource: Datasource = {
     ...baseDatasource,
     ...sdk.datasources.mergeConfigs(dataSourceBody, baseDatasource),
   }
+
+  // this block is specific to GSheets, if no auth set, set it back
+  const auth = baseDatasource.config?.auth
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
     datasource.config!.auth = auth
@@ -204,7 +218,7 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   const db = context.getAppDB()
 
   // Get all internal tables
-  const internalTables = await db.allDocs(
+  const internalTables = await db.allDocs<Table>(
     getTableParams(null, {
       include_docs: true,
     })
@@ -212,8 +226,8 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
 
   // Filter by datasource and return the docs.
   const datasourceTableDocs = internalTables.rows.reduce(
-    (acc: any, table: any) => {
-      if (table.doc.sourceId == datasourceId) {
+    (acc: Table[], table) => {
+      if (table.doc?.sourceId == datasourceId) {
         acc.push(table.doc)
       }
       return acc
@@ -254,9 +268,9 @@ export async function destroy(ctx: UserCtx) {
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
     await destroyInternalTablesBySourceId(datasourceId)
   } else {
-    const queries = await db.allDocs(getQueryParams(datasourceId))
+    const queries = await db.allDocs<RowValue>(getQueryParams(datasourceId))
     await db.bulkDocs(
-      queries.rows.map((row: any) => ({
+      queries.rows.map(row => ({
         _id: row.id,
         _rev: row.value.rev,
         _deleted: true,

packages/server/src/api/controllers/integration.ts

@@ -1,7 +1,10 @@
 import { getDefinition, getDefinitions } from "../../integrations"
 import { SourceName, UserCtx } from "@budibase/types"
 
-const DISABLED_EXTERNAL_INTEGRATIONS = [SourceName.AIRTABLE]
+const DISABLED_EXTERNAL_INTEGRATIONS = [
+  SourceName.AIRTABLE,
+  SourceName.BUDIBASE,
+]
 
 export async function fetch(ctx: UserCtx) {
   const definitions = await getDefinitions()

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -1,8 +1,8 @@
 import {
   DEFAULT_BB_DATASOURCE_ID,
-  DEFAULT_INVENTORY_TABLE_ID,
   DEFAULT_EMPLOYEE_TABLE_ID,
   DEFAULT_EXPENSES_TABLE_ID,
+  DEFAULT_INVENTORY_TABLE_ID,
   DEFAULT_JOBS_TABLE_ID,
 } from "../../constants"
 import { importToRows } from "../../api/controllers/table/utils"
@@ -15,19 +15,21 @@ import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
+  Datasource,
   FieldType,
   RelationshipType,
   Row,
+  SourceName,
   Table,
   TableSchema,
   TableSourceType,
 } from "@budibase/types"
 
-const defaultDatasource = {
+const defaultDatasource: Datasource = {
   _id: DEFAULT_BB_DATASOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
   name: "Sample Data",
-  source: "BUDIBASE",
+  source: SourceName.BUDIBASE,
   config: {},
 }
 

packages/server/src/db/utils.ts

@@ -1,13 +1,15 @@
 import newid from "./newid"
 import { db as dbCore } from "@budibase/backend-core"
 import {
-  FieldType,
+  DatabaseQueryOpts,
+  Datasource,
   DocumentType,
   FieldSchema,
-  RelationshipFieldMetadata,
-  VirtualDocumentType,
+  FieldType,
   INTERNAL_TABLE_SOURCE_ID,
-  DatabaseQueryOpts,
+  RelationshipFieldMetadata,
+  SourceName,
+  VirtualDocumentType,
 } from "@budibase/types"
 
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -20,11 +22,11 @@ export const enum AppStatus {
   DEPLOYED = "published",
 }
 
-export const BudibaseInternalDB = {
+export const BudibaseInternalDB: Datasource = {
   _id: INTERNAL_TABLE_SOURCE_ID,
   type: dbCore.BUDIBASE_DATASOURCE_TYPE,
   name: "Budibase DB",
-  source: "BUDIBASE",
+  source: SourceName.BUDIBASE,
   config: {},
 }
 

packages/server/src/integrations/index.ts

@@ -37,6 +37,7 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
   [SourceName.REDIS]: redis.schema,
   [SourceName.SNOWFLAKE]: snowflake.schema,
   [SourceName.ORACLE]: undefined,
+  [SourceName.BUDIBASE]: undefined,
 }
 
 const INTEGRATIONS: Record<SourceName, any> = {
@@ -56,6 +57,7 @@ const INTEGRATIONS: Record<SourceName, any> = {
   [SourceName.REDIS]: redis.integration,
   [SourceName.SNOWFLAKE]: snowflake.integration,
   [SourceName.ORACLE]: undefined,
+  [SourceName.BUDIBASE]: undefined,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

packages/server/src/sdk/app/applications/import.ts

@@ -85,7 +85,9 @@ async function getImportableDocuments(db: Database) {
   const docPromises = []
   for (let docType of DocumentTypesToImport) {
     docPromises.push(
-      db.allDocs(dbCore.getDocParams(docType, null, { include_docs: true }))
+      db.allDocs<Document>(
+        dbCore.getDocParams(docType, null, { include_docs: true })
+      )
     )
   }
   // map the responses to the document itself

packages/server/src/sdk/app/datasources/datasources.ts

@@ -229,7 +229,7 @@ export async function removeSecretSingle(datasource: Datasource) {
 }
 
 export function mergeConfigs(update: Datasource, old: Datasource) {
-  if (!update.config) {
+  if (!update.config || !old.config) {
     return update
   }
   // specific to REST datasources, fix the auth configs again if required

packages/types/src/api/web/app/datasource.ts

@@ -32,9 +32,7 @@ export interface FetchDatasourceInfoResponse {
   tableNames: string[]
 }
 
-export interface UpdateDatasourceRequest extends Datasource {
-  datasource: Datasource
-}
+export interface UpdateDatasourceRequest extends Datasource {}
 
 export interface BuildSchemaFromSourceRequest {
   tablesFilter?: string[]

packages/types/src/documents/app/datasource.ts

@@ -6,6 +6,9 @@ export interface Datasource extends Document {
   type: string
   name?: string
   source: SourceName
+  // this is a googlesheets specific property which
+  // can be found in the GSheets schema - pertains to SSO creds
+  auth?: { type: string }
   // the config is defined by the schema
   config?: Record<string, any>
   plus?: boolean
@@ -36,6 +39,12 @@ export interface RestAuthConfig {
   config: RestBasicAuthConfig | RestBearerAuthConfig
 }
 
+export interface DynamicVariable {
+  name: string
+  queryId: string
+  value: string
+}
+
 export interface RestConfig {
   url: string
   rejectUnauthorized: boolean
@@ -47,11 +56,5 @@ export interface RestConfig {
   staticVariables: {
     [key: string]: string
   }
-  dynamicVariables: [
-    {
-      name: string
-      queryId: string
-      value: string
-    }
-  ]
+  dynamicVariables: DynamicVariable[]
 }

packages/types/src/documents/pouch.ts

@@ -5,15 +5,15 @@ export interface RowValue {
   deleted: boolean
 }
 
-export interface RowResponse<T extends Document> {
+export interface RowResponse<T extends Document | RowValue> {
   id: string
   key: string
   error: string
-  value: T | RowValue
+  value: T
   doc?: T
 }
 
-export interface AllDocsResponse<T extends Document> {
+export interface AllDocsResponse<T extends Document | RowValue> {
   offset: number
   total_rows: number
   rows: RowResponse<T>[]

packages/types/src/sdk/datasources.ts

@@ -56,6 +56,7 @@ export enum SourceName {
   FIRESTORE = "FIRESTORE",
   REDIS = "REDIS",
   SNOWFLAKE = "SNOWFLAKE",
+  BUDIBASE = "BUDIBASE",
 }
 
 export enum IncludeRelationship {

packages/types/src/sdk/db.ts

@@ -1,5 +1,11 @@
 import type Nano from "@budibase/nano"
-import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../"
+import {
+  AllDocsResponse,
+  AnyDocument,
+  Document,
+  RowValue,
+  ViewTemplateOpts,
+} from "../"
 import { Writable } from "stream"
 
 export enum SearchIndex {
@@ -135,7 +141,7 @@ export interface Database {
     opts?: DatabasePutOpts
   ): Promise<Nano.DocumentInsertResponse>
   bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
-  allDocs<T extends Document>(
+  allDocs<T extends Document | RowValue>(
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>>
   query<T extends Document>(