Updating datasource save API to allow specifying a filter for which tables to fetch and then linking this up with the new frontend.

packages/builder/src/builderStore/datasource.js

@@ -23,10 +23,11 @@ function prepareData(config) {
   return datasource
 }
 
-export async function saveDatasource(config, skipFetch = false) {
+export async function saveDatasource(config, { skipFetch, tablesFilter } = {}) {
   const datasource = prepareData(config)
   // Create datasource
-  const resp = await datasources.save(datasource, !skipFetch && datasource.plus)
+  const fetchSchema = !skipFetch && datasource.plus
+  const resp = await datasources.save(datasource, { fetchSchema, tablesFilter })
 
   // update the tables incase datasource plus
   await tables.fetch()

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -62,7 +62,11 @@
       if (!datasource.name) {
         datasource.name = name
       }
-      const resp = await save(datasource)
+      const opts = {}
+      if (datasourcePlus && selectedTables) {
+        opts.tablesFilter = selectedTables
+      }
+      const resp = await save(datasource, opts)
       $goto(`./datasource/${resp._id}`)
       notifications.success("Datasource created successfully.")
     } catch (err) {
@@ -80,7 +84,7 @@
     if (!connected) {
       return false
     }
-    if (datasourcePlus) {
+    if (datasourcePlus && !fetchTableStep) {
       notifications.success("Connected to datasource successfully.")
       const info = await getDatasourceInfo(datasource)
       tableList = info.tableNames

packages/builder/src/stores/backend/datasources.js

@@ -57,7 +57,10 @@ export function createDatasourcesStore() {
     return updateDatasource(response)
   }
 
-  const save = async (body, fetchSchema = false) => {
+  const save = async (body, { fetchSchema, tablesFilter } = {}) => {
+    if (fetchSchema == null) {
+      fetchSchema = false
+    }
     let response
     if (body._id) {
       response = await API.updateDatasource(body)
@@ -65,6 +68,7 @@ export function createDatasourcesStore() {
       response = await API.createDatasource({
         datasource: body,
         fetchSchema,
+        tablesFilter,
       })
     }
     return updateDatasource(response)

packages/frontend-core/src/api/datasources.js

@@ -26,13 +26,16 @@ export const buildDatasourceEndpoints = API => ({
    * Creates a datasource
    * @param datasource the datasource to create
    * @param fetchSchema whether to fetch the schema or not
+   * @param tablesFilter a list of tables to actually fetch rather than simply
+   * all that are accessible.
    */
-  createDatasource: async ({ datasource, fetchSchema }) => {
+  createDatasource: async ({ datasource, fetchSchema, tablesFilter }) => {
     return await API.post({
       url: "/api/datasources",
       body: {
         datasource,
         fetchSchema,
+        tablesFilter,
       },
     })
   },

packages/server/src/api/controllers/datasource.ts

@@ -103,6 +103,22 @@ async function buildSchemaHelper(datasource: Datasource) {
   return { tables: connector.tables, error }
 }
 
+async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
+  let { tables, error } = await buildSchemaHelper(datasource)
+  let finalTables = tables
+  if (filter) {
+    finalTables = {}
+    for (let key in tables) {
+      if (
+        filter.some((filter: any) => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        finalTables[key] = tables[key]
+      }
+    }
+  }
+  return { tables: finalTables, error }
+}
+
 export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
@@ -180,37 +196,22 @@ export async function information(
 
 export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  let { tables, error } = await buildSchemaHelper(datasource)
-  if (tablesFilter) {
-    if (!datasource.entities) {
-      datasource.entities = {}
-    }
-    for (let key in tables) {
-      if (
-        tablesFilter.some(
-          (filter: any) => filter.toLowerCase() === key.toLowerCase()
-        )
-      ) {
-        datasource.entities[key] = tables[key]
-      }
-    }
-  } else {
-    datasource.entities = tables
-  }
+  const { tables, error } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
   const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const response: any = { datasource: cleanedDatasource }
+  const res: any = { datasource: cleanedDatasource }
   if (error) {
-    response.error = error
+    res.error = error
   }
-  ctx.body = response
+  ctx.body = res
 }
 
 /**
@@ -320,6 +321,7 @@ export async function save(
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
+  const tablesFilter = ctx.request.body.tablesFilter
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
@@ -329,7 +331,10 @@ export async function save(
 
   let schemaError = null
   if (fetchSchema) {
-    const { tables, error } = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     schemaError = error
     datasource.entities = tables
     setDefaultDisplayColumns(datasource)

packages/types/src/api/web/app/datasource.ts

@@ -12,6 +12,7 @@ export interface UpdateDatasourceResponse {
 export interface CreateDatasourceRequest {
   datasource: Datasource
   fetchSchema?: boolean
+  tablesFilter: string[]
 }
 
 export interface VerifyDatasourceRequest {