Merge branch 'master' into fix/overflow-invoices

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.14.3",
+  "version": "2.14.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6
+Subproject commit bcd86d9034ba954f013da4c10171bf495ab88189

packages/builder/src/api.js

@@ -5,7 +5,7 @@ import {
 } from "@budibase/frontend-core"
 import { store } from "./builderStore"
 import { get } from "svelte/store"
-import { auth } from "./stores/portal"
+import { auth, navigation } from "./stores/portal"
 
 export const API = createAPIClient({
   attachHeaders: headers => {
@@ -45,4 +45,15 @@ export const API = createAPIClient({
       }
     }
   },
+  onMigrationDetected: appId => {
+    const updatingUrl = `/builder/app/updating/${appId}`
+
+    if (window.location.pathname === updatingUrl) {
+      return
+    }
+
+    get(navigation).goto(
+      `${updatingUrl}?returnUrl=${encodeURIComponent(window.location.pathname)}`
+    )
+  },
 })

packages/builder/src/pages/builder/_layout.svelte

@@ -1,6 +1,6 @@
 <script>
   import { isActive, redirect, params } from "@roxi/routify"
-  import { admin, auth, licensing } from "stores/portal"
+  import { admin, auth, licensing, navigation } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
   import { API } from "api"
@@ -17,6 +17,8 @@
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
+  navigation.actions.init($redirect)
+
   const validateTenantId = async () => {
     const host = window.location.host
     if (host.includes("localhost:") || !baseUrl) {

packages/builder/src/pages/builder/app/updating/[application].svelte

@@ -0,0 +1,19 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { redirect, params } from "@roxi/routify"
+
+  import { API } from "api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    // For some reason routify params is not stripping the ? properly, so we need to check both with and without ?
+    const returnUrl = $params.returnUrl || $params["?returnUrl"]
+    $redirect(returnUrl)
+  }
+</script>
+
+<Updating {isMigrationDone} {onMigrationDone} />

packages/builder/src/stores/portal/index.js

@@ -16,5 +16,6 @@ export { environment } from "./environment"
 export { menu } from "./menu"
 export { auditLogs } from "./auditLogs"
 export { features } from "./features"
+export { navigation } from "./navigation"
 
 export const sideBarCollapsed = writable(false)

packages/builder/src/stores/portal/navigation.js

@@ -0,0 +1,31 @@
+import { writable } from "svelte/store"
+
+export function createNavigationStore() {
+  const store = writable({
+    initialisated: false,
+    goto: undefined,
+  })
+  const { set, subscribe } = store
+
+  const init = gotoFunc => {
+    if (typeof gotoFunc !== "function") {
+      throw new Error(
+        `gotoFunc must be a function, found a "${typeof gotoFunc}" instead`
+      )
+    }
+
+    set({
+      initialisated: true,
+      goto: gotoFunc,
+    })
+  }
+
+  return {
+    subscribe,
+    actions: {
+      init,
+    },
+  }
+}
+
+export const navigation = createNavigationStore()

packages/client/package.json

@@ -37,7 +37,6 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "regexparam": "^1.3.0",
     "sanitize-html": "^2.7.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",

packages/client/src/api/api.js

@@ -77,4 +77,10 @@ export const API = createAPIClient({
     // Log all errors to console
     console.warn(`[Client] HTTP ${status} on ${method}:${url}\n\t${message}`)
   },
+  onMigrationDetected: _appId => {
+    if (!window.MIGRATING_APP) {
+      // We will force a reload, that will display the updating screen until the migration is running
+      window.location.reload()
+    }
+  },
 })

packages/client/src/components/UpdatingApp.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Updating } from "@budibase/frontend-core"
+  import { API } from "../api"
+
+  async function isMigrationDone() {
+    const response = await API.getMigrationStatus()
+    return response.migrated
+  }
+
+  async function onMigrationDone() {
+    window.location.reload()
+  }
+</script>
+
+<div class="updating">
+  <Updating {isMigrationDone} {onMigrationDone} />
+</div>
+
+<style>
+  .updating {
+    font-family: var(--font-sans);
+  }
+</style>

packages/client/src/index.js

@@ -1,4 +1,5 @@
 import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
 import {
   builderStore,
   appStore,
@@ -52,6 +53,13 @@ const loadBudibase = async () => {
     window["##BUDIBASE_APP_EMBEDDED##"] === "true"
   )
 
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
   // Fetch environment info
   if (!get(environmentStore)?.loaded) {
     await environmentStore.actions.fetchEnvironment()

packages/frontend-core/src/api/index.js

@@ -33,6 +33,7 @@ import { buildEnvironmentVariableEndpoints } from "./environmentVariables"
 import { buildEventEndpoints } from "./events"
 import { buildAuditLogsEndpoints } from "./auditLogs"
 import { buildLogsEndpoints } from "./logs"
+import { buildMigrationEndpoints } from "./migrations"
 
 /**
  * Random identifier to uniquely identify a session in a tab. This is
@@ -298,6 +299,7 @@ export const createAPIClient = config => {
     ...buildEventEndpoints(API),
     ...buildAuditLogsEndpoints(API),
     ...buildLogsEndpoints(API),
+    ...buildMigrationEndpoints(API),
     viewV2: buildViewV2Endpoints(API),
   }
 }

packages/frontend-core/src/api/migrations.js

@@ -0,0 +1,10 @@
+export const buildMigrationEndpoints = API => ({
+  /**
+   * Gets the info about the current app migration
+   */
+  getMigrationStatus: async () => {
+    return await API.get({
+      url: "/api/migrations/status",
+    })
+  },
+})

packages/frontend-core/src/components/Updating.svelte

@@ -0,0 +1,79 @@
+<script>
+  export let isMigrationDone
+  export let onMigrationDone
+  export let timeoutSeconds = 10 // 3 minutes
+
+  const loadTime = Date.now()
+  let timedOut = false
+
+  async function checkMigrationsFinished() {
+    setTimeout(async () => {
+      const isMigrated = await isMigrationDone()
+
+      const timeoutMs = timeoutSeconds * 1000
+      if (!isMigrated) {
+        if (loadTime + timeoutMs > Date.now()) {
+          return checkMigrationsFinished()
+        }
+
+        return migrationTimeout()
+      }
+
+      onMigrationDone()
+    }, 1000)
+  }
+
+  checkMigrationsFinished()
+
+  function migrationTimeout() {
+    timedOut = true
+  }
+</script>
+
+<div class="loading" class:timeout={timedOut}>
+  <span class="header">
+    {#if !timedOut}
+      System update
+    {:else}
+      Something went wrong!
+    {/if}
+  </span>
+  <span class="subtext">
+    {#if !timedOut}
+      Please wait and we will be back in a second!
+    {:else}
+      An error occurred, please try again later.
+      <br />
+      Contact
+      <a href="https://budibase.com/support/" target="_blank">support</a> if the
+      issue persists.
+    {/if}</span
+  >
+</div>
+
+<style>
+  .loading {
+    display: flex;
+    justify-content: center;
+    flex-direction: column;
+    gap: var(--spacing-l);
+    height: 100vh;
+    text-align: center;
+    font-size: 18px;
+  }
+  .header {
+    font-weight: 700;
+  }
+  .timeout .header {
+    color: rgb(196, 46, 46);
+  }
+  .subtext {
+    font-size: 16px;
+    color: var(--grey-7);
+  }
+
+  .subtext a {
+    color: var(--grey-7);
+    font-weight: 700;
+  }
+</style>

packages/frontend-core/src/components/index.js

@@ -3,4 +3,5 @@ export { default as TestimonialPage } from "./TestimonialPage.svelte"
 export { default as Testimonial } from "./Testimonial.svelte"
 export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
+export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"

packages/server/nodemon.json

@@ -7,7 +7,7 @@
     "../shared-core",
     "../string-templates"
   ],
-  "ext": "js,ts,json",
+  "ext": "js,ts,json,svelte",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
   "exec": "yarn build && node ./dist/index.js"
 }

packages/server/src/api/controllers/datasource.ts

@@ -14,10 +14,7 @@ import {
   DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
-  IntegrationBase,
-  Schema,
   SourceName,
-  Table,
   UpdateDatasourceResponse,
   UserCtx,
   VerifyDatasourceRequest,
@@ -28,65 +25,6 @@ import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
-async function getConnector(
-  datasource: Datasource
-): Promise<IntegrationBase | DatasourcePlus> {
-  const Connector = await getIntegration(datasource.source)
-  // can't enrich if it doesn't have an ID yet
-  if (datasource._id) {
-    datasource = await sdk.datasources.enrich(datasource)
-  }
-  // Connect to the DB and build the schema
-  return new Connector(datasource.config)
-}
-
-async function getAndMergeDatasource(datasource: Datasource) {
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
-  return await sdk.datasources.enrich(enrichedDatasource)
-}
-
-async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
-  const connector = (await getConnector(datasource)) as DatasourcePlus
-  return await connector.buildSchema(
-    datasource._id!,
-    datasource.entities! as Record<string, Table>
-  )
-}
-
-async function buildFilteredSchema(
-  datasource: Datasource,
-  filter?: string[]
-): Promise<Schema> {
-  let schema = await buildSchemaHelper(datasource)
-  if (!filter) {
-    return schema
-  }
-
-  let filteredSchema: Schema = { tables: {}, errors: {} }
-  for (let key in schema.tables) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.tables[key] = schema.tables[key]
-    }
-  }
-
-  for (let key in schema.errors) {
-    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
-      filteredSchema.errors[key] = schema.errors[key]
-    }
-  }
-  return filteredSchema
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.datasources.fetch()
 }
@@ -95,8 +33,10 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = await getConnector(enrichedDatasource)
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
   }
@@ -112,8 +52,12 @@ export async function information(
   ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
   const { datasource } = ctx.request.body
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
+    datasource
+  )
+  const connector = (await sdk.datasources.getConnector(
+    enrichedDatasource
+  )) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -128,7 +72,10 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   const tablesFilter = ctx.request.body.tablesFilter
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+    datasource,
+    tablesFilter
+  )
   datasource.entities = tables
 
   setDefaultDisplayColumns(datasource)
@@ -280,7 +227,10 @@ export async function save(
 
   let errors: Record<string, string> = {}
   if (fetchSchema) {
-    const schema = await buildFilteredSchema(datasource, tablesFilter)
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
     datasource.entities = schema.tables
     setDefaultDisplayColumns(datasource)
     errors = schema.errors
@@ -384,8 +334,10 @@ export async function query(ctx: UserCtx) {
 
 export async function getExternalSchema(ctx: UserCtx) {
   const datasource = await sdk.datasources.get(ctx.params.datasourceId)
-  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const enrichedDatasource = await sdk.datasources.getAndMergeDatasource(
-  const connector = await getConnector(enrichedDatasource)
+    datasource
+  )
+  const connector = await sdk.datasources.getConnector(enrichedDatasource)
 
   if (!connector.getExternalSchema) {
     ctx.throw(400, "Datasource does not support exporting external schema")

packages/server/src/api/controllers/static/index.ts

@@ -25,8 +25,12 @@ import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../../appMigrations"
 
-const send = require("koa-send")
+import send from "koa-send"
 
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
@@ -125,7 +129,26 @@ export const deleteObjects = async function (ctx: Ctx) {
   )
 }
 
+const requiresMigration = async (ctx: Ctx) => {
+  const appId = context.getAppId()
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestMigration = getLatestMigrationId()
+  if (!latestMigration) {
+    return false
+  }
+
+  const latestMigrationApplied = await getAppMigrationVersion(appId)
+
+  const requiresMigrations = latestMigrationApplied !== latestMigration
+  return requiresMigrations
+}
+
 export const serveApp = async function (ctx: Ctx) {
+  const needMigrations = await requiresMigration(ctx)
+
   const bbHeaderEmbed =
     ctx.request.get("x-budibase-embed")?.toLowerCase() === "true"
 
@@ -145,8 +168,8 @@ export const serveApp = async function (ctx: Ctx) {
     let appId = context.getAppId()
 
     if (!env.isJest()) {
-      const App = require("./templates/BudibaseApp.svelte").default
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+      const App = require("./templates/BudibaseApp.svelte").default
       const { head, html, css } = App.render({
         metaImage:
           branding?.metaImageUrl ||
@@ -167,6 +190,7 @@ export const serveApp = async function (ctx: Ctx) {
           config?.logoUrl !== ""
             ? objectStore.getGlobalFileUrl("settings", "logoUrl")
             : "",
+        appMigrating: needMigrations,
       })
       const appHbs = loadHandlebarsFile(appHbsPath)
       ctx.body = await processString(appHbs, {
@@ -273,7 +297,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
-      return
     }
     try {
       const s3 = new AWS.S3({

packages/server/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -8,6 +8,7 @@
 
   export let clientLibPath
   export let usedPlugins
+  export let appMigrating
 </script>
 
 <svelte:head>
@@ -110,6 +111,11 @@
   <script type="application/javascript">
     window.INIT_TIME = Date.now()
   </script>
+  {#if appMigrating}
+    <script type="application/javascript">
+      window.MIGRATING_APP = true
+    </script>
+  {/if}
   <script type="application/javascript" src={clientLibPath}>
   </script>
   <!-- Custom components need inserted after the core client library -->

packages/server/src/appMigrations/index.ts

@@ -17,7 +17,7 @@ export const getLatestMigrationId = () =>
     .sort()
     .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
 
 export async function checkMissingMigrations(
   ctx: UserCtx,

packages/server/src/integrations/postgres.ts

@@ -362,8 +362,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         })
       }
 
-      let finalizedTables = finaliseExternalTables(tables, entities)
+      const finalizedTables = finaliseExternalTables(tables, entities)
-      let errors = checkExternalTables(finalizedTables)
+      const errors = checkExternalTables(finalizedTables)
       return { tables: finalizedTables, errors }
     } catch (err) {
       // @ts-ignore

packages/server/src/jsRunner.ts

@@ -12,9 +12,13 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
+        tracer.trace<any>("runJS.setupTracker", {}, span => {
           const bbCtx = context.getCurrentContext()
           if (bbCtx) {
             if (!bbCtx.jsExecutionTracker) {
+              span?.addTags({
+                createdExecutionTracker: true,
+              })
               bbCtx.jsExecutionTracker =
                 timers.ExecutionTimeTracker.withLimit(perRequestLimit)
             }
@@ -27,22 +31,33 @@ export function init() {
             // We call checkLimit() here to prevent paying the cost of creating
             // a new VM context below when we don't need to.
             bbCtx.jsExecutionTracker.checkLimit()
-          track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
+            track = bbCtx.jsExecutionTracker.track.bind(
+              bbCtx.jsExecutionTracker
+            )
           }
+        })
       }
 
-      ctx = {
+      ctx = tracer.trace("runJS.ctxClone", {}, span => {
+        return {
           ...ctx,
           alert: undefined,
           setInterval: undefined,
           setTimeout: undefined,
         }
+      })
+
+      tracer.trace("runJS.vm.createContext", {}, span => {
         vm.createContext(ctx)
+      })
+
       return track(() =>
+        tracer.trace("runJS.vm.runInNewContext", {}, span =>
           vm.runInNewContext(js, ctx, {
             timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
           })
         )
+      )
     })
   })
 }

packages/server/src/sdk/app/datasources/index.ts

@@ -1,5 +1,7 @@
 import * as datasources from "./datasources"
+import * as plus from "./plus"
 
 export default {
   ...datasources,
+  ...plus,
 }

packages/server/src/sdk/app/datasources/plus.ts

@@ -0,0 +1,62 @@
+import {
+  Datasource,
+  DatasourcePlus,
+  IntegrationBase,
+  Schema,
+} from "@budibase/types"
+import * as datasources from "./datasources"
+import { getIntegration } from "../../../integrations"
+
+export async function buildFilteredSchema(
+  datasource: Datasource,
+  filter?: string[]
+): Promise<Schema> {
+  const schema = await buildSchemaHelper(datasource)
+  if (!filter) {
+    return schema
+  }
+
+  let filteredSchema: Schema = { tables: {}, errors: {} }
+  for (let key in schema.tables) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.tables[key] = schema.tables[key]
+    }
+  }
+
+  for (let key in schema.errors) {
+    if (filter.some(filter => filter.toLowerCase() === key.toLowerCase())) {
+      filteredSchema.errors[key] = schema.errors[key]
+    }
+  }
+  return filteredSchema
+}
+
+async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const externalSchema = await connector.buildSchema(
+    datasource._id!,
+    datasource.entities!
+  )
+  return externalSchema
+}
+
+export async function getConnector(
+  datasource: Datasource
+): Promise<IntegrationBase | DatasourcePlus> {
+  const Connector = await getIntegration(datasource.source)
+  // can't enrich if it doesn't have an ID yet
+  if (datasource._id) {
+    datasource = await datasources.enrich(datasource)
+  }
+  // Connect to the DB and build the schema
+  return new Connector(datasource.config)
+}
+
+export async function getAndMergeDatasource(datasource: Datasource) {
+  if (datasource._id) {
+    const existingDatasource = await datasources.get(datasource._id)
+
+    datasource = datasources.mergeConfigs(datasource, existingDatasource)
+  }
+  return await datasources.enrich(datasource)
+}

yarn.lock

@@ -18312,11 +18312,6 @@ regexparam@2.0.1:
   resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-2.0.1.tgz#c912f5dae371e3798100b3c9ce22b7414d0889fa"
   integrity sha512-zRgSaYemnNYxUv+/5SeoHI0eJIgTL/A2pUtXUPLHQxUldagouJ9p+K6IbIZ/JiQuCEv2E2B1O11SjVQy3aMCkw==
 
-regexparam@^1.3.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
-  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
-
 regexpu-core@^5.3.1:
   version "5.3.1"
   resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.3.1.tgz#66900860f88def39a5cb79ebd9490e84f17bcdfb"