Adding in basic implementation of variable usage, getting from pro and enriching through new datasource SDK.

packages/backend-core/src/security/encryption.ts

@@ -12,9 +12,9 @@ function stretchString(string: string, salt: Buffer) {
   return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512")
 }
 
-export function encrypt(input: string) {
+export function encrypt(input: string, secret: string | undefined = SECRET) {
   const salt = crypto.randomBytes(RANDOM_BYTES)
-  const stretched = stretchString(SECRET!, salt)
+  const stretched = stretchString(secret!, salt)
   const cipher = crypto.createCipheriv(ALGO, stretched, salt)
   const base = cipher.update(input)
   const final = cipher.final()
@@ -22,10 +22,10 @@ export function encrypt(input: string) {
   return `${salt.toString("hex")}${SEPARATOR}${encrypted}`
 }
 
-export function decrypt(input: string) {
+export function decrypt(input: string, secret: string | undefined = SECRET) {
   const [salt, encrypted] = input.split(SEPARATOR)
   const saltBuffer = Buffer.from(salt, "hex")
-  const stretched = stretchString(SECRET!, saltBuffer)
+  const stretched = stretchString(secret!, saltBuffer)
   const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer)
   const base = decipher.update(Buffer.from(encrypted, "hex"))
   const final = decipher.final()

packages/builder/src/stores/portal/envVars.js

@@ -2,19 +2,19 @@ import { writable } from "svelte/store"
 import { API } from "api"
 
 export function createEnvVarsStore() {
-    const { subscribe, set, update } = writable([])
+  const { subscribe, set, update } = writable([])
 
-    async function load() {
+  async function load() {
-        const envVars = await API.fetchEnvVars()
+    const envVars = await API.fetchEnvVars()
 
-        let testVars = ['blah', 'blah123']
+    let testVars = ["blah", "blah123"]
-        set(testVars)
+    set(testVars)
-    }
+  }
 
-    return {
+  return {
-        subscribe,
+    subscribe,
-        load,
+    load,
-    }
+  }
 }
 
 export const envVars = createEnvVarsStore()

packages/frontend-core/src/api/envVars.js

@@ -1,11 +1,11 @@
 export const buildEnvironmentVariableEndpoints = API => ({
-    /**
+  /**
-     * Fetches a list of environment variables
+   * Fetches a list of environment variables
-     */
+   */
-    fetchEnvVars: async () => {
+  fetchEnvVars: async () => {
-        return await API.get({
+    return await API.get({
-            url: `/api/env/variables`,
+      url: `/api/env/variables`,
-            json: false,
+      json: false,
-        })
+    })
-    }
+  },
 })

packages/server/__mocks__/node-fetch.ts

@@ -1,5 +1,7 @@
+// @ts-ignore
 import fs from "fs"
 module FetchMock {
+  // @ts-ignore
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
 

packages/server/src/api/controllers/datasource.ts

@@ -13,6 +13,8 @@ import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
 import { BBContext, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { cloneDeep } from "lodash/fp"
 
 export async function fetch(ctx: BBContext) {
   // Get internal tables
@@ -61,7 +63,7 @@ export async function fetch(ctx: BBContext) {
 
 export async function buildSchemaFromDb(ctx: BBContext) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -149,8 +151,8 @@ async function invalidateVariables(
 export async function update(ctx: BBContext) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
+  let datasource = await sdk.datasources.get(datasourceId)
-  const auth = datasource.config.auth
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -162,7 +164,7 @@ export async function update(ctx: BBContext) {
   datasource = { ...datasource, ...dataSourceBody }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -255,7 +257,7 @@ export async function destroy(ctx: BBContext) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -313,6 +315,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
+  datasource = await sdk.datasources.enrichDatasourceWithValues(datasource)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)

packages/server/src/api/controllers/query/index.ts

@@ -7,6 +7,7 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import { QUERY_THREAD_TIMEOUT } from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: QUERY_THREAD_TIMEOUT || 10000,
@@ -81,7 +82,7 @@ export async function save(ctx: any) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
   if (!query._id) {
@@ -126,9 +127,9 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(ctx.request.body.datasourceId, {
-
+    withEnvVars: true,
-  const datasource = await db.get(ctx.request.body.datasourceId)
+  })
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
@@ -201,7 +202,9 @@ async function execute(
   const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId, {
+    withEnvVars: true,
+  })
 
   let authConfigCtx: any = {}
   if (!opts.isAutomation) {
@@ -266,18 +269,18 @@ export async function executeV2(
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
-  const dynamicVariables = datasource.config.dynamicVariables
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    datasource.config.dynamicVariables = dynamicVariables.filter(
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
       (dv: any) => dv.queryId !== queryId
     )
     await db.put(datasource)
 
     // invalidate the deleted variables
-    const variablesToDelete = dynamicVariables.filter(
+    const variablesToDelete = dynamicVariables!.filter(
       (dv: any) => dv.queryId === queryId
     )
     await invalidateDynamicVariables(variablesToDelete)
@@ -289,7 +292,7 @@ export async function destroy(ctx: any) {
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
 import { removeKeyNumbering } from "./utils"
+import sdk from "../../../sdk"
 
 export interface ManyRelationship {
   tableId?: string
@@ -664,8 +665,7 @@ export class ExternalRequest {
       throw "Unable to run without a table name"
     }
     if (!this.datasource) {
-      const db = context.getAppDB()
+      this.datasource = await sdk.datasources.get(datasourceId!)
-      this.datasource = await db.get(datasourceId)
       if (!this.datasource || !this.datasource.entities) {
         throw "No tables found, fetch tables before query."
       }

packages/server/src/api/controllers/row/external.ts

@@ -19,6 +19,7 @@ import {
   Table,
   Datasource,
 } from "@budibase/types"
+import sdk from "../../../sdk"
 
 export async function handleRequest(
   operation: Operation,
@@ -179,10 +180,9 @@ export async function validate(ctx: BBContext) {
 
 export async function exportRows(ctx: BBContext) {
   const { datasourceId } = breakExternalTableId(ctx.params.tableId)
-  const db = context.getAppDB()
   const format = ctx.query.format
   const { columns } = ctx.request.body
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
@@ -225,8 +225,7 @@ export async function fetchEnrichedRow(ctx: BBContext) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = context.getAppDB()
+  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
-  const datasource: Datasource = await db.get(datasourceId)
   if (!tableName) {
     ctx.throw(400, "Unable to find table.")
   }

packages/server/src/api/controllers/row/utils.ts

@@ -8,6 +8,7 @@ export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
 import { Ctx } from "@budibase/types"
+import sdk from "../../../sdk"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -21,8 +22,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
-  const db = context.getAppDB()
+  const datasource = await sdk.datasources.get(datasourceId)
-  const datasource = await db.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 

packages/server/src/api/controllers/static/index.ts

@@ -1,20 +1,21 @@
 require("svelte/register")
 
-const send = require("koa-send")
+import { resolve, join } from "../../../utilities/centralPath"
-const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
-const { processString } = require("@budibase/string-templates")
+import { processString } from "@budibase/string-templates"
-const {
+import {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
-} = require("../../../utilities/fileSystem")
+} from "../../../utilities/fileSystem"
-const env = require("../../../environment")
+import env from "../../../environment"
-const { DocumentType } = require("../../../db/utils")
+import { DocumentType } from "../../../db/utils"
-const { context, objectStore, utils } = require("@budibase/backend-core")
+import { context, objectStore, utils } from "@budibase/backend-core"
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const fs = require("fs")
+import fs from "fs"
+import sdk from "../../../sdk"
+const send = require("koa-send")
 
 async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await objectStore.upload({
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on
@@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) {
 }
 
 export const getSignedUploadURL = async function (ctx: any) {
-  const database = context.getAppDB()
-
   // Ensure datasource is valid
   let datasource
   try {
     const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
+    datasource = await sdk.datasources.get(datasourceId, { withEnvVars: true })
     if (!datasource) {
       ctx.throw(400, "The specified datasource could not be found")
     }
@@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) {
   // Determine type of datasource and generate signed URL
   let signedUrl
   let publicUrl
-  const awsRegion = datasource?.config?.region || "eu-west-1"
+  const awsRegion = (datasource?.config?.region || "eu-west-1") as string
-  if (datasource.source === "S3") {
+  if (datasource?.source === "S3") {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
@@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) {
     try {
       const s3 = new AWS.S3({
         region: awsRegion,
-        accessKeyId: datasource?.config?.accessKeyId,
+        accessKeyId: datasource?.config?.accessKeyId as string,
-        secretAccessKey: datasource?.config?.secretAccessKey,
+        secretAccessKey: datasource?.config?.secretAccessKey as string,
         apiVersion: "2006-03-01",
         signatureVersion: "v4",
       })

packages/server/src/api/controllers/table/external.ts

@@ -219,7 +219,7 @@ export async function save(ctx: BBContext) {
   }
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   if (!datasource.entities) {
     datasource.entities = {}
   }
@@ -322,15 +322,17 @@ export async function destroy(ctx: BBContext) {
   const datasourceId = getDatasourceId(tableToDelete)
 
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   const tables = datasource.entities
 
   const operation = Operation.DELETE_TABLE
-  await makeTableRequest(datasource, operation, tableToDelete, tables)
+  if (tables) {
+    await makeTableRequest(datasource, operation, tableToDelete, tables)
+    cleanupRelationships(tableToDelete, tables)
+    delete tables[tableToDelete.name]
+    datasource.entities = tables
+  }
 
-  cleanupRelationships(tableToDelete, tables)
-
-  delete datasource.entities[tableToDelete.name]
   await db.put(datasource)
 
   return tableToDelete

packages/server/src/integrations/base/query.ts

@@ -1,10 +1,12 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { getIntegration } = require("../index")
+import { getIntegration } from "../index"
+import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
+  datasource = await sdk.datasources.enrichDatasourceWithValues(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -0,0 +1,24 @@
+import { environmentVariables } from "@budibase/pro"
+import { context } from "@budibase/backend-core"
+import { processObject } from "@budibase/string-templates"
+import { Datasource } from "@budibase/types"
+import { cloneDeep } from "lodash/fp"
+
+export async function enrichDatasourceWithValues(datasource: Datasource) {
+  const cloned = cloneDeep(datasource)
+  const envVars = await environmentVariables.fetchValues()
+  return (await processObject(cloned, envVars)) as Datasource
+}
+
+export async function get(
+  datasourceId: string,
+  opts?: { withEnvVars: boolean }
+): Promise<Datasource> {
+  const appDb = context.getAppDB()
+  const datasource = await appDb.get(datasourceId)
+  if (opts?.withEnvVars) {
+    return await enrichDatasourceWithValues(datasource)
+  } else {
+    return datasource
+  }
+}

packages/server/src/sdk/app/datasources/index.ts

@@ -0,0 +1,5 @@
+import * as datasources from "./datasources"
+
+export default {
+  ...datasources,
+}

packages/server/src/sdk/app/tables/index.ts

@@ -6,6 +6,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { Table, Database } from "@budibase/types"
+import datasources from "../datasources"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
@@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
   }))
 }
 
-async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+async function getAllExternalTables(
+  datasourceId: any
+): Promise<Record<string, Table>> {
   const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await datasources.get(datasourceId, { withEnvVars: true })
   if (!datasource || !datasource.entities) {
     throw "Datasource is not configured fully."
   }
@@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
+    const datasource = await datasources.get(datasourceId!)
     const table = await getExternalTable(datasourceId, tableName)
     return { ...table, sql: isSQL(datasource) }
   } else {

packages/server/src/sdk/index.ts

@@ -2,6 +2,7 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as datasources } from "./app/datasources"
 import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
@@ -12,6 +13,7 @@ const sdk = {
   applications,
   rows,
   users,
+  datasources,
 }
 
 // default export for TS

packages/server/src/threads/query.ts

@@ -6,6 +6,7 @@ import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
 
 const { isSQL } = require("../integrations/utils")
@@ -166,7 +167,9 @@ class QueryRunner {
   async runAnotherQuery(queryId: string, parameters: any) {
     const db = context.getAppDB()
     const query = await db.get(queryId)
-    const datasource = await db.get(query.datasourceId)
+    const datasource = await sdk.datasources.get(query.datasourceId, {
+      withEnvVars: true,
+    })
     return new QueryRunner(
       {
         datasource,

packages/types/src/documents/app/datasource.ts

@@ -8,7 +8,7 @@ export interface Datasource extends Document {
   source: SourceName
   // the config is defined by the schema
   config?: {
-    [key: string]: string | number | boolean
+    [key: string]: string | number | boolean | any[]
   }
   plus?: boolean
   entities?: {