Merge remote-tracking branch 'origin/master' into global-bindings

hosting/couchdb/runner.sh

@@ -76,6 +76,6 @@ done
 
 # CouchDB needs the `_users` and `_replicator` databases to exist before it will
 # function correctly, so we create them here.
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
-curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
 sleep infinity

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.14.3",
+  "version": "2.14.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit b11e6b47370d9b77c63648b45929c86bfed6360c
+Subproject commit bcd86d9034ba954f013da4c10171bf495ab88189

packages/backend-core/src/middleware/authenticated.ts

@@ -172,11 +172,8 @@ export default function (
         tracer.setUser({
           id: user?._id,
           tenantId: user?.tenantId,
-          admin: user?.admin,
-          builder: user?.builder,
           budibaseAccess: user?.budibaseAccess,
           status: user?.status,
-          roles: user?.roles,
         })
       }
 

packages/builder/src/api.js

@@ -5,7 +5,7 @@ import {
 } from "@budibase/frontend-core"
 import { store } from "./builderStore"
 import { get } from "svelte/store"
-import { auth } from "./stores/portal"
+import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
   attachHeaders: headers => {
@@ -45,4 +45,15 @@ export const API = createAPIClient({
       }
     }
   },
+  onMigrationDetected: appId => {
+    const updatingUrl = `/builder/app/updating/${appId}`
+
+    if (window.location.pathname === updatingUrl) {
+      return
+    }
+
+    get(navigation).goto(
+      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location.pathname)}`
+    )
+  },
 })

packages/builder/src/pages/builder/_layout.svelte

@@ -1,6 +1,6 @@
 <script>
   import { isActive, redirect, params } from "@roxi/routify"
-  import { admin, auth, licensing } from "stores/portal"
+  import { admin, auth, licensing, navigation } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
   import { API } from "api"
@@ -17,6 +17,8 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
+  navigation.actions.init($redirect)
+
   const validateTenantId = async () => {
     const host = window.location.host
     if (host.includes("localhost:") || !baseUrl) {

packages/builder/src/pages/builder/app/updating/[application].svelte

@@ -0,0 +1,19 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { redirect, params } from "@roxi/routify"
+
+  import { API } from "api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
+    const returnUrl = $params.returnUrl || $params["?returnUrl"]
+    $redirect(returnUrl)
+  }
+</script>
+
+<Updating {isMigrationDone} {onMigrationDone} />

packages/builder/src/stores/portal/index.js

@@ -16,5 +16,6 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+export { navigation } from "./navigation"
 
 export const sideBarCollapsed = writable(false)

packages/builder/src/stores/portal/navigation.js

@@ -0,0 +1,31 @@
+import { writable } from "svelte/store"
+
+export function createNavigationStore() {
+  const store = writable({
+    initialisated: false,
+    goto: undefined,
+  })
+  const { set, subscribe } = store
+
+  const init = gotoFunc => {
+    if (typeof gotoFunc !== "function") {
+      throw new Error(
+        `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`
+      )
+    }
+
+    set({
+      initialisated: true,
+      goto: gotoFunc,
+    })
+  }
+
+  return {
+    subscribe,
+    actions: {
+      init,
+    },
+  }
+}
+
+export const navigation = createNavigationStore()

packages/client/package.json

@@ -37,7 +37,6 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "regexparam": "^1.3.0",
     "sanitize-html": "^2.7.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",

packages/client/src/api/api.js

@@ -77,4 +77,10 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
+  onMigrationDetected: _appId => {
+    if (!window.MIGRATING_APP) {
+      // We will force a reload, that will display the updating screen until the migration is running
+      window.location.reload()
+    }
+  },
 })

packages/client/src/components/UpdatingApp.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { API } from "../api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    window.location.reload()
+  }
+</script>
+
+<div class="updating">
+  <Updating {isMigrationDone} {onMigrationDone} />
+</div>
+
+<style>
+  .updating {
+    font-family: var(--font-sans);
+  }
+</style>

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -14,6 +14,7 @@
 
   const { fetchDatasourceSchema } = getContext("sdk")
   const component = getContext("component")
+  const context = getContext("context")
 
   // Set current step context to force child form to use it
   const currentStep = writable(1)
@@ -157,18 +158,23 @@
             <BlockComponent type="heading" props={{ text: step.title }} />
           </BlockComponent>
           <BlockComponent type="text" props={{ text: step.desc }} order={1} />
-          <BlockComponent type="fieldgroup" order={2}>
-            {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
-              {#if getComponentForField(field)}
-                <BlockComponent
-                  type={getComponentForField(field)}
-                  props={getPropsForField(field)}
-                  order={fieldIdx}
-                  interactive
-                  name={field.field}
-                />
-              {/if}
-            {/each}
+          <BlockComponent type="container" order={2}>
+            <div
+              class="form-block fields"
+              class:mobile={$context.device.mobile}
+            >
+              {#each step.fields as field, fieldIdx (`${field.field || field.name}_${stepIdx}_${fieldIdx}`)}
+                {#if getComponentForField(field)}
+                  <BlockComponent
+                    type={getComponentForField(field)}
+                    props={getPropsForField(field)}
+                    order={fieldIdx}
+                    interactive
+                    name={field.field}
+                  />
+                {/if}
+              {/each}
+            </div>
           </BlockComponent>
           <BlockComponent
             type="buttongroup"
@@ -185,3 +191,14 @@
     {/each}
   </BlockComponent>
 </FormBlockWrapper>
+
+<style>
+  .fields {
+    display: grid;
+    grid-template-columns: repeat(6, 1fr);
+    gap: 8px 16px;
+  }
+  .fields.mobile :global(.spectrum-Form-item) {
+    grid-column: span 6 !important;
+  }
+</style>

packages/client/src/index.js

@@ -1,4 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
 import {
   builderStore,
   appStore,
@@ -52,6 +53,13 @@ const loadBudibase = async () => {
     window["##BUDIBASE_APP_EMBEDDED##"] === "true"
   )
 
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
   // Fetch environment info
   if (!get(environmentStore)?.loaded) {
     await environmentStore.actions.fetchEnvironment()

packages/frontend-core/src/api/index.js

@@ -33,6 +33,7 @@ import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
 import { buildLogsEndpoints } from "./logs"
+import { buildMigrationEndpoints } from "./migrations"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -298,6 +299,7 @@ export const createAPIClient = config => {
     ...buildEventEndpoints(API),
     ...buildAuditLogsEndpoints(API),
     ...buildLogsEndpoints(API),
+    ...buildMigrationEndpoints(API),
     viewV2: buildViewV2Endpoints(API),
   }
 }

packages/frontend-core/src/api/migrations.js

@@ -0,0 +1,10 @@
+export const buildMigrationEndpoints = API => ({
+  /**
+   * Gets the info about the current app migration
+   */
+  getMigrationStatus: async () => {
+    return await API.get({
+      url: "/api/migrations/status",
+    })
+  },
+})

packages/frontend-core/src/components/Updating.svelte

@@ -0,0 +1,79 @@
+<script>
+  export let isMigrationDone
+  export let onMigrationDone
+  export let timeoutSeconds = 10 // 3 minutes
+
+  const loadTime = Date.now()
+  let timedOut = false
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const isMigrated = await isMigrationDone()
+
+      const timeoutMs = timeoutSeconds * 1000
+      if (!isMigrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      onMigrationDone()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    timedOut = true
+  }
+</script>
+
+<div class="loading" class:timeout={timedOut}>
+  <span class="header">
+    {#if !timedOut}
+      System update
+    {:else}
+      Something went wrong!
+    {/if}
+  </span>
+  <span class="subtext">
+    {#if !timedOut}
+      Please wait and we will be back in a second!
+    {:else}
+      An error occurred, please try again later.
+      <br />
+      Contact
+      <a href="https://budibase.com/support/" target="_blank">support</a> if the
+      issue persists.
+    {/if}</span
+  >
+</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-l);
+    height: 100vh;
+    text-align: center;
+    font-size: 18px;
+  }
+  .header {
+    font-weight: 700;
+  }
+  .timeout .header {
+    color: rgb(196, 46, 46);
+  }
+  .subtext {
+    font-size: 16px;
+    color: var(--grey-7);
+  }
+
+  .subtext a {
+    color: var(--grey-7);
+    font-weight: 700;
+  }
+</style>

packages/frontend-core/src/components/index.js

@@ -3,4 +3,5 @@ export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
+export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"

packages/server/nodemon.json

@@ -7,7 +7,7 @@
     "../shared-core",
     "../string-templates"
   ],
-  "ext": "js,ts,json",
+  "ext": "js,ts,json,svelte",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

packages/server/src/api/controllers/datasource.ts

@@ -14,10 +14,7 @@ import {
   DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
-  IntegrationBase,
-  Schema,
   SourceName,
-  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -28,65 +25,6 @@ import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await sdk.datasources.enrich(enrichedDatasource)
-}
-
-async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(
-    datasource._id!,
-    datasource.entities! as Record<string, Table>
-  )
-}
-
-async function buildFilteredSchema(
-  datasource: Datasource,
-  filter?: string[]
-): Promise<Schema> {
-  let schema = await buildSchemaHelper(datasource)
-  if (!filter) {
-    return schema
-  }
-
-  let filteredSchema: Schema = { tables: {}, errors: {} }
-  for (let key in schema.tables) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.tables[key] = schema.tables[key]
-    }
-  }
-
-  for (let key in schema.errors) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.errors[key] = schema.errors[key]
-    }
-  }
-  return filteredSchema
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
 }
@@ -95,8 +33,10 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -112,8 +52,12 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = (await sdk.datasources.getConnector(
+    enrichedDatasource
+  )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -128,7 +72,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+    datasource,
+    tablesFilter
+  )
   datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
@@ -280,7 +227,10 @@ export async function save(
 
   let errors: Record<string, string> = {}
   if (fetchSchema) {
-    const schema = await buildFilteredSchema(datasource, tablesFilter)
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     datasource.entities = schema.tables
     setDefaultDisplayColumns(datasource)
     errors = schema.errors
@@ -384,8 +334,10 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
-  const connector = await getConnector(enrichedDatasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")

packages/server/src/api/controllers/static/index.ts

@@ -25,8 +25,12 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../../appMigrations"
 
-const send = require("koa-send")
+import send from "koa-send"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -125,7 +129,26 @@ export const deleteObjects = async function (ctx: Ctx) {
   )
 }
 
+const requiresMigration = async (ctx: Ctx) => {
+  const appId = context.getAppId()
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestMigration = getLatestMigrationId()
+  if (!latestMigration) {
+    return false
+  }
+
+  const latestMigrationApplied = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestMigrationApplied !== latestMigration
+  return requiresMigrations
+}
+
 export const serveApp = async function (ctx: Ctx) {
+  const needMigrations = await requiresMigration(ctx)
+
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -145,8 +168,8 @@ export const serveApp = async function (ctx: Ctx) {
     let appId = context.getAppId()
 
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         metaImage:
           branding?.metaImageUrl ||
@@ -167,6 +190,7 @@ export const serveApp = async function (ctx: Ctx) {
           config?.logoUrl !== ""
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
+        appMigrating: needMigrations,
       })
       const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
@@ -273,7 +297,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
-      return
     }
     try {
       const s3 = new AWS.S3({

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -8,6 +8,7 @@
 
   export let clientLibPath
   export let usedPlugins
+  export let appMigrating
 </script>
 
 <svelte:head>
@@ -110,6 +111,11 @@
   <script type="application/javascript">
     window.INIT_TIME = Date.now()
   </script>
+  {#if appMigrating}
+    <script type="application/javascript">
+      window.MIGRATING_APP = true
+    </script>
+  {/if}
   <script type="application/javascript" src={clientLibPath}>
   </script>
   <!-- Custom components need inserted after the core client library -->

packages/server/src/appMigrations/index.ts

@@ -17,7 +17,7 @@ export const getLatestMigrationId = () =>
     .sort()
     .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
 
 export async function checkMissingMigrations(
   ctx: UserCtx,

packages/server/src/ddApm.ts

@@ -3,5 +3,9 @@ import apm from "dd-trace"
 // enable APM if configured
 if (process.env.DD_APM_ENABLED) {
   console.log("Starting dd-trace")
-  apm.init()
+  apm.init({
+    // @ts-ignore for some reason dd-trace types don't include this options,
+    // even though it's spoken about in the docs.
+    debug: process.env.DD_ENV === "qa",
+  })
 }

packages/server/src/integration-test/postgres.spec.ts

@@ -1118,4 +1118,76 @@ describe("postgres integrations", () => {
       })
     })
   })
+
+  describe("Integration compatibility with postgres search_path", () => {
+    let client: Client, pathDatasource: Datasource
+    const schema1 = "test1",
+      schema2 = "test-2"
+
+    beforeAll(async () => {
+      const dsConfig = await databaseTestProviders.postgres.getDsConfig()
+      const dbConfig = dsConfig.config!
+
+      client = new Client(dbConfig)
+      await client.connect()
+      await client.query(`CREATE SCHEMA "${schema1}";`)
+      await client.query(`CREATE SCHEMA "${schema2}";`)
+
+      const pathConfig: any = {
+        ...dsConfig,
+        config: {
+          ...dbConfig,
+          schema: `${schema1}, ${schema2}`,
+        },
+      }
+      pathDatasource = await config.api.datasource.create(pathConfig)
+    })
+
+    afterAll(async () => {
+      await client.query(`DROP SCHEMA "${schema1}" CASCADE;`)
+      await client.query(`DROP SCHEMA "${schema2}" CASCADE;`)
+      await client.end()
+    })
+
+    it("discovers tables from any schema in search path", async () => {
+      await client.query(
+        `CREATE TABLE "${schema1}".table1 (id1 SERIAL PRIMARY KEY);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".table2 (id2 SERIAL PRIMARY KEY);`
+      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: pathDatasource,
+      })
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames).toEqual(
+        expect.arrayContaining(["table1", "table2"])
+      )
+    })
+
+    it("does not mix columns from different tables", async () => {
+      const repeated_table_name = "table_same_name"
+      await client.query(
+        `CREATE TABLE "${schema1}".${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
+      )
+      await client.query(
+        `CREATE TABLE "${schema2}".${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
+      )
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${pathDatasource._id}/schema`,
+        {
+          tablesFilter: [repeated_table_name],
+        }
+      )
+      expect(response.status).toBe(200)
+      expect(
+        response.body.datasource.entities[repeated_table_name].schema
+      ).toBeDefined()
+      const schema =
+        response.body.datasource.entities[repeated_table_name].schema
+      expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
+    })
+  })
 })

packages/server/src/integrations/postgres.ts

@@ -159,7 +159,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   JOIN pg_index ON pg_class.oid = pg_index.indrelid AND pg_index.indisprimary
   JOIN pg_attribute ON pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = ANY(pg_index.indkey)
   JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
-  WHERE pg_namespace.nspname = '${this.config.schema}';
+  WHERE pg_namespace.nspname = ANY(current_schemas(false))
+  AND pg_table_is_visible(pg_class.oid);
   `
 
   ENUM_VALUES = () => `
@@ -219,8 +220,12 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     if (!this.config.schema) {
       this.config.schema = "public"
     }
-    await this.client.query(`SET search_path TO "${this.config.schema}"`)
-    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`
+    const search_path = this.config.schema
+      .split(",")
+      .map(item => `"${item.trim()}"`)
+    await this.client.query(`SET search_path TO ${search_path.join(",")};`)
+    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = ANY(current_schemas(false))
+    AND pg_table_is_visible(to_regclass(table_schema || '.' || table_name));`
     this.open = true
   }
 
@@ -362,8 +367,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         })
       }
 
-      let finalizedTables = finaliseExternalTables(tables, entities)
-      let errors = checkExternalTables(finalizedTables)
+      const finalizedTables = finaliseExternalTables(tables, entities)
+      const errors = checkExternalTables(finalizedTables)
       return { tables: finalizedTables, errors }
     } catch (err) {
       // @ts-ignore

packages/server/src/jsRunner.ts

@@ -12,36 +12,51 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
-        const bbCtx = context.getCurrentContext()
-        if (bbCtx) {
-          if (!bbCtx.jsExecutionTracker) {
-            bbCtx.jsExecutionTracker =
-              timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+        tracer.trace<any>("runJS.setupTracker", {}, span => {
+          const bbCtx = context.getCurrentContext()
+          if (bbCtx) {
+            if (!bbCtx.jsExecutionTracker) {
+              span?.addTags({
+                createdExecutionTracker: true,
+              })
+              bbCtx.jsExecutionTracker =
+                timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+            }
+            span?.addTags({
+              js: {
+                limitMS: bbCtx.jsExecutionTracker.limitMs,
+                elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+              },
+            })
+            // We call checkLimit() here to prevent paying the cost of creating
+            // a new VM context below when we don't need to.
+            bbCtx.jsExecutionTracker.checkLimit()
+            track = bbCtx.jsExecutionTracker.track.bind(
+              bbCtx.jsExecutionTracker
+            )
           }
-          span?.addTags({
-            js: {
-              limitMS: bbCtx.jsExecutionTracker.limitMs,
-              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-            },
-          })
-          // We call checkLimit() here to prevent paying the cost of creating
-          // a new VM context below when we don't need to.
-          bbCtx.jsExecutionTracker.checkLimit()
-          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
-        }
+        })
       }
 
-      ctx = {
-        ...ctx,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-      vm.createContext(ctx)
+      ctx = tracer.trace("runJS.ctxClone", {}, span => {
+        return {
+          ...ctx,
+          alert: undefined,
+          setInterval: undefined,
+          setTimeout: undefined,
+        }
+      })
+
+      tracer.trace("runJS.vm.createContext", {}, span => {
+        vm.createContext(ctx)
+      })
+
       return track(() =>
-        vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-        })
+        tracer.trace("runJS.vm.runInNewContext", {}, span =>
+          vm.runInNewContext(js, ctx, {
+            timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+          })
+        )
       )
     })
   })

packages/server/src/sdk/app/datasources/index.ts

@@ -1,5 +1,7 @@
 import * as datasources from "./datasources"
+import * as plus from "./plus"
 
 export default {
   ...datasources,
+  ...plus,
 }

packages/server/src/sdk/app/datasources/plus.ts

@@ -0,0 +1,62 @@
+import {
+  Datasource,
+  DatasourcePlus,
+  IntegrationBase,
+  Schema,
+} from "@budibase/types"
+import * as datasources from "./datasources"
+import { getIntegration } from "../../../integrations"
+
+export async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  const schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
+    }
+  }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
+}
+
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const externalSchema = await connector.buildSchema(
+    datasource._id!,
+    datasource.entities!
+  )
+  return externalSchema
+}
+
+export async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+export async function getAndMergeDatasource(datasource: Datasource) {
+  if (datasource._id) {
+    const existingDatasource = await datasources.get(datasource._id)
+
+    datasource = datasources.mergeConfigs(datasource, existingDatasource)
+  }
+  return await datasources.enrich(datasource)
+}

yarn.lock

@@ -18312,11 +18312,6 @@ regexparam@2.0.1:
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
   integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
 
-regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"