Merge pull request #8937 from Budibase/fix/8896

Automation logs availability after unpublish

packages/server/src/api/controllers/application.ts

@@ -26,7 +26,6 @@ import {
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
-
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import {
   clientLibraryPath,
@@ -39,18 +38,22 @@ import {
   backupClientLibrary,
   revertClientLibrary,
 } from "../../utilities/fileSystem/clientLibrary"
-import { syncGlobalUsers } from "./user"
 import { cleanupAutomations } from "../../automations/utils"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas, groups } from "@budibase/pro"
-import { App, Layout, Screen, MigrationType } from "@budibase/types"
+import {
+  App,
+  Layout,
+  Screen,
+  MigrationType,
+  BBContext,
+  Database,
+} from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
 import sdk from "../../sdk"
 
-const URL_REGEX_SLASH = /\/|\\/g
-
 // utility function, need to do away with this
 async function getLayouts() {
   const db = context.getAppDB()
@@ -74,29 +77,18 @@ async function getScreens() {
   ).rows.map((row: any) => row.doc)
 }
 
-function getUserRoleId(ctx: any) {
+function getUserRoleId(ctx: BBContext) {
-  return !ctx.user.role || !ctx.user.role._id
+  return !ctx.user?.role || !ctx.user.role._id
     ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
-export const getAppUrl = (ctx: any) => {
+function checkAppUrl(
-  // construct the url
+  ctx: BBContext,
-  let url
+  apps: App[],
-  if (ctx.request.body.url) {
+  url: string,
-    // if the url is provided, use that
+  currentAppId?: string
-    url = encodeURI(ctx.request.body.url)
+) {
-  } else if (ctx.request.body.name) {
-    // otherwise use the name
-    url = encodeURI(`${ctx.request.body.name}`)
-  }
-  if (url) {
-    url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
-  }
-  return url
-}
-
-const checkAppUrl = (ctx: any, apps: any, url: any, currentAppId?: string) => {
   if (currentAppId) {
     apps = apps.filter((app: any) => app.appId !== currentAppId)
   }
@@ -105,12 +97,12 @@ const checkAppUrl = (ctx: any, apps: any, url: any, currentAppId?: string) => {
   }
 }
 
-const checkAppName = (
+function checkAppName(
-  ctx: any,
+  ctx: BBContext,
-  apps: any,
+  apps: App[],
-  name: any,
+  name: string,
   currentAppId?: string
-) => {
+) {
   // TODO: Replace with Joi
   if (!name) {
     ctx.throw(400, "Name is required")
@@ -165,14 +157,14 @@ async function createInstance(template: any, includeSampleData: boolean) {
   return { _id: appId }
 }
 
-const addDefaultTables = async (db: any) => {
+async function addDefaultTables(db: Database) {
   const defaultDbDocs = buildDefaultDocs()
 
   // add in the default db data docs - tables, datasource, rows and links
   await db.bulkDocs([...defaultDbDocs])
 }
 
-export const fetch = async (ctx: any) => {
+export async function fetch(ctx: BBContext) {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = (await dbCore.getAllApps({ dev, all })) as App[]
@@ -197,7 +189,7 @@ export const fetch = async (ctx: any) => {
   ctx.body = await checkAppMetadata(apps)
 }
 
-export const fetchAppDefinition = async (ctx: any) => {
+export async function fetchAppDefinition(ctx: BBContext) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -212,7 +204,7 @@ export const fetchAppDefinition = async (ctx: any) => {
   }
 }
 
-export const fetchAppPackage = async (ctx: any) => {
+export async function fetchAppPackage(ctx: BBContext) {
   const db = context.getAppDB()
   let application = await db.get(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
@@ -222,7 +214,7 @@ export const fetchAppPackage = async (ctx: any) => {
   application.usedPlugins = enrichPluginURLs(application.usedPlugins)
 
   // Only filter screens if the user is not a builder
-  if (!(ctx.user.builder && ctx.user.builder.global)) {
+  if (!(ctx.user?.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
     const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
@@ -236,11 +228,12 @@ export const fetchAppPackage = async (ctx: any) => {
   }
 }
 
-const performAppCreate = async (ctx: any) => {
+async function performAppCreate(ctx: BBContext) {
-  const apps = await dbCore.getAllApps({ dev: true })
+  const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const name = ctx.request.body.name
+  const name = ctx.request.body.name,
+    possibleUrl = ctx.request.body.url
   checkAppName(ctx, apps, name)
-  const url = getAppUrl(ctx)
+  const url = sdk.applications.getAppUrl({ name, url: possibleUrl })
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -331,7 +324,7 @@ const performAppCreate = async (ctx: any) => {
   return newApplication
 }
 
-const creationEvents = async (request: any, app: App) => {
+async function creationEvents(request: any, app: App) {
   let creationFns: ((app: App) => Promise<void>)[] = []
 
   const body = request.body
@@ -356,7 +349,7 @@ const creationEvents = async (request: any, app: App) => {
   }
 }
 
-const appPostCreate = async (ctx: any, app: App) => {
+async function appPostCreate(ctx: BBContext, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -377,7 +370,7 @@ const appPostCreate = async (ctx: any, app: App) => {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
           // delete the app
-          // skip pre and post steps as no rows have been added to quotas yet
+          // skip pre- and post-steps as no rows have been added to quotas yet
           ctx.params.appId = app.appId
           await destroyApp(ctx)
         }
@@ -387,7 +380,7 @@ const appPostCreate = async (ctx: any, app: App) => {
   }
 }
 
-export const create = async (ctx: any) => {
+export async function create(ctx: BBContext) {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
   await cache.bustCache(cache.CacheKey.CHECKLIST)
@@ -397,14 +390,15 @@ export const create = async (ctx: any) => {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export const update = async (ctx: any) => {
+export async function update(ctx: BBContext) {
-  const apps = await dbCore.getAllApps({ dev: true })
+  const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
-  const name = ctx.request.body.name
+  const name = ctx.request.body.name,
+    possibleUrl = ctx.request.body.url
   if (name) {
     checkAppName(ctx, apps, name, ctx.params.appId)
   }
-  const url = getAppUrl(ctx)
+  const url = sdk.applications.getAppUrl({ name, url: possibleUrl })
   if (url) {
     checkAppUrl(ctx, apps, url, ctx.params.appId)
     ctx.request.body.url = url
@@ -416,7 +410,7 @@ export const update = async (ctx: any) => {
   ctx.body = app
 }
 
-export const updateClient = async (ctx: any) => {
+export async function updateClient(ctx: BBContext) {
   // Get current app version
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -440,7 +434,7 @@ export const updateClient = async (ctx: any) => {
   ctx.body = app
 }
 
-export const revertClient = async (ctx: any) => {
+export async function revertClient(ctx: BBContext) {
   // Check app can be reverted
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -466,12 +460,15 @@ export const revertClient = async (ctx: any) => {
   ctx.body = app
 }
 
-const destroyApp = async (ctx: any) => {
+async function destroyApp(ctx: BBContext) {
   let appId = ctx.params.appId
   let isUnpublish = ctx.query && ctx.query.unpublish
 
   if (isUnpublish) {
     appId = dbCore.getProdAppID(appId)
+    const devAppId = dbCore.getDevAppID(appId)
+    // sync before removing the published app
+    await sdk.applications.syncApp(devAppId)
   }
 
   const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
@@ -501,12 +498,12 @@ const destroyApp = async (ctx: any) => {
   return result
 }
 
-const preDestroyApp = async (ctx: any) => {
+async function preDestroyApp(ctx: BBContext) {
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 
-const postDestroyApp = async (ctx: any) => {
+async function postDestroyApp(ctx: BBContext) {
   const rowCount = ctx.rowCount
   await groups.cleanupApp(ctx.params.appId)
   if (rowCount) {
@@ -514,7 +511,7 @@ const postDestroyApp = async (ctx: any) => {
   }
 }
 
-export const destroy = async (ctx: any) => {
+export async function destroy(ctx: BBContext) {
   await preDestroyApp(ctx)
   const result = await destroyApp(ctx)
   await postDestroyApp(ctx)
@@ -522,62 +519,16 @@ export const destroy = async (ctx: any) => {
   ctx.body = result
 }
 
-export const sync = async (ctx: any, next: any) => {
+export async function sync(ctx: BBContext) {
-  if (env.DISABLE_AUTO_PROD_APP_SYNC) {
-    ctx.status = 200
-    ctx.body = {
-      message:
-        "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
-    }
-    return next()
-  }
-
   const appId = ctx.params.appId
-  if (!dbCore.isDevAppID(appId)) {
-    ctx.throw(400, "This action cannot be performed for production apps")
-  }
-
-  // replicate prod to dev
-  const prodAppId = dbCore.getProdAppID(appId)
-
-  // specific case, want to make sure setup is skipped
-  const prodDb = context.getProdAppDB({ skip_setup: true })
-  const exists = await prodDb.exists()
-  if (!exists) {
-    // the database doesn't exist. Don't replicate
-    ctx.status = 200
-    ctx.body = {
-      message: "App sync not required, app not deployed.",
-    }
-    return next()
-  }
-
-  const replication = new dbCore.Replication({
-    source: prodAppId,
-    target: appId,
-  })
-  let error
   try {
-    await replication.replicate(replication.appReplicateOpts())
+    ctx.body = await sdk.applications.syncApp(appId)
-  } catch (err) {
+  } catch (err: any) {
-    error = err
+    ctx.throw(err.status || 400, err.message)
-  } finally {
-    await replication.close()
-  }
-
-  // sync the users
-  await syncGlobalUsers()
-
-  if (error) {
-    ctx.throw(400, error)
-  } else {
-    ctx.body = {
-      message: "App sync completed successfully.",
-    }
   }
 }
 
-export const updateAppPackage = async (appPackage: any, appId: any) => {
+export async function updateAppPackage(appPackage: any, appId: any) {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)
@@ -598,7 +549,7 @@ export const updateAppPackage = async (appPackage: any, appId: any) => {
   })
 }
 
-const migrateAppNavigation = async () => {
+async function migrateAppNavigation() {
   const db = context.getAppDB()
   const existing: App = await db.get(DocumentType.APP_METADATA)
   const layouts: Layout[] = await getLayouts()

packages/server/src/api/controllers/cloud.ts

@@ -22,6 +22,7 @@ async function createApp(appName: string, appDirectory: string) {
       },
     },
   }
+  // @ts-ignore
   return create(ctx)
 }
 

packages/server/src/api/controllers/user.ts

@@ -1,12 +1,7 @@
-import {
+import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
-  generateUserMetadataID,
-  getUserMetadataParams,
-  generateUserFlagID,
-} from "../../db/utils"
 import { InternalTables } from "../../db/utils"
 import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
-import { isEqual } from "lodash"
 import {
   context,
   constants,
@@ -14,59 +9,7 @@ import {
   db as dbCore,
 } from "@budibase/backend-core"
 import { BBContext, User } from "@budibase/types"
-
+import sdk from "../../sdk"
-async function rawMetadata() {
-  const db = context.getAppDB()
-  return (
-    await db.allDocs(
-      getUserMetadataParams(null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc)
-}
-
-function combineMetadataAndUser(user: any, metadata: any) {
-  // skip users with no access
-  if (user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC) {
-    return null
-  }
-  delete user._rev
-  const metadataId = generateUserMetadataID(user._id)
-  const newDoc = {
-    ...user,
-    _id: metadataId,
-    tableId: InternalTables.USER_METADATA,
-  }
-  const found = Array.isArray(metadata)
-    ? metadata.find(doc => doc._id === metadataId)
-    : metadata
-  // copy rev over for the purposes of equality check
-  if (found) {
-    newDoc._rev = found._rev
-  }
-  if (found == null || !isEqual(newDoc, found)) {
-    return {
-      ...found,
-      ...newDoc,
-    }
-  }
-  return null
-}
-
-export async function syncGlobalUsers() {
-  // sync user metadata
-  const db = context.getAppDB()
-  const [users, metadata] = await Promise.all([getGlobalUsers(), rawMetadata()])
-  const toWrite = []
-  for (let user of users) {
-    const combined = await combineMetadataAndUser(user, metadata)
-    if (combined) {
-      toWrite.push(combined)
-    }
-  }
-  await db.bulkDocs(toWrite)
-}
 
 export async function syncUser(ctx: BBContext) {
   let deleting = false,
@@ -123,7 +66,7 @@ export async function syncUser(ctx: BBContext) {
           metadata.roleId = roleId
         }
         let combined = !deleting
-          ? combineMetadataAndUser(user, metadata)
+          ? sdk.users.combineMetadataAndUser(user, metadata)
           : {
               ...metadata,
               status: constants.UserStatus.INACTIVE,
@@ -143,7 +86,7 @@ export async function syncUser(ctx: BBContext) {
 
 export async function fetchMetadata(ctx: BBContext) {
   const global = await getGlobalUsers()
-  const metadata = await rawMetadata()
+  const metadata = await sdk.users.rawUserMetadata()
   const users = []
   for (let user of global) {
     // find the metadata that matches up to the global ID

packages/server/src/api/routes/tests/application.spec.js

@@ -17,7 +17,6 @@ const {
   checkBuilderEndpoint,
 } = require("./utilities/TestFunctions")
 const setup = require("./utilities")
-const { basicScreen, basicLayout } = setup.structures
 const { AppStatus } = require("../../../db/utils")
 const { events } = require("@budibase/backend-core")
 

packages/server/src/migrations/functions/appUrls.ts

@@ -1,5 +1,5 @@
 import { db as dbCore } from "@budibase/backend-core"
-import { getAppUrl } from "../../api/controllers/application"
+import sdk from "../../sdk"
 
 /**
  * Date:
@@ -20,14 +20,7 @@ export const run = async (appDb: any) => {
   }
 
   if (!metadata.url) {
-    const context = {
+    metadata.url = sdk.applications.getAppUrl({ name: metadata.name })
-      request: {
-        body: {
-          name: metadata.name,
-        },
-      },
-    }
-    metadata.url = getAppUrl(context)
     console.log(`Adding url to app: ${metadata.url}`)
     await appDb.put(metadata)
   }

packages/server/src/sdk/app/applications/index.ts

@@ -0,0 +1,7 @@
+import * as sync from "./sync"
+import * as utils from "./utils"
+
+export default {
+  ...sync,
+  ...utils,
+}

packages/server/src/sdk/app/applications/sync.ts

@@ -0,0 +1,53 @@
+import env from "../../../environment"
+import { db as dbCore, context } from "@budibase/backend-core"
+import sdk from "../../"
+
+export async function syncApp(appId: string) {
+  if (env.DISABLE_AUTO_PROD_APP_SYNC) {
+    return {
+      message:
+        "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable.",
+    }
+  }
+
+  if (dbCore.isProdAppID(appId)) {
+    throw new Error("This action cannot be performed for production apps")
+  }
+
+  // replicate prod to dev
+  const prodAppId = dbCore.getProdAppID(appId)
+
+  // specific case, want to make sure setup is skipped
+  const prodDb = context.getProdAppDB({ skip_setup: true })
+  const exists = await prodDb.exists()
+  if (!exists) {
+    // the database doesn't exist. Don't replicate
+    return {
+      message: "App sync not required, app not deployed.",
+    }
+  }
+
+  const replication = new dbCore.Replication({
+    source: prodAppId,
+    target: appId,
+  })
+  let error
+  try {
+    await replication.replicate(replication.appReplicateOpts())
+  } catch (err) {
+    error = err
+  } finally {
+    await replication.close()
+  }
+
+  // sync the users
+  await sdk.users.syncGlobalUsers()
+
+  if (error) {
+    throw error
+  } else {
+    return {
+      message: "App sync completed successfully.",
+    }
+  }
+}

packages/server/src/sdk/app/applications/utils.ts

@@ -0,0 +1,17 @@
+const URL_REGEX_SLASH = /\/|\\/g
+
+export function getAppUrl(opts?: { name?: string; url?: string }) {
+  // construct the url
+  let url
+  if (opts?.url) {
+    // if the url is provided, use that
+    url = encodeURI(opts?.url)
+  } else if (opts?.name) {
+    // otherwise use the name
+    url = encodeURI(`${opts?.name}`)
+  }
+  if (url) {
+    url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
+  }
+  return url as string
+}

packages/server/src/sdk/index.ts

@@ -1,15 +1,16 @@
 import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
+import { default as applications } from "./app/applications"
+import { default as users } from "./users"
 
 const sdk = {
   backups,
   tables,
   automations,
+  applications,
+  users,
 }
 
 // default export for TS
 export default sdk
-
-// default export for JS
-module.exports = sdk

packages/server/src/sdk/users/index.ts

@@ -0,0 +1,5 @@
+import * as utils from "./utils"
+
+export default {
+  ...utils,
+}

packages/server/src/sdk/users/utils.ts

@@ -0,0 +1,64 @@
+import { getGlobalUsers } from "../../utilities/global"
+import { context, roles as rolesCore } from "@budibase/backend-core"
+import {
+  generateUserMetadataID,
+  getUserMetadataParams,
+  InternalTables,
+} from "../../db/utils"
+import { isEqual } from "lodash"
+
+export function combineMetadataAndUser(user: any, metadata: any) {
+  // skip users with no access
+  if (user.roleId === rolesCore.BUILTIN_ROLE_IDS.PUBLIC) {
+    return null
+  }
+  delete user._rev
+  const metadataId = generateUserMetadataID(user._id)
+  const newDoc = {
+    ...user,
+    _id: metadataId,
+    tableId: InternalTables.USER_METADATA,
+  }
+  const found = Array.isArray(metadata)
+    ? metadata.find(doc => doc._id === metadataId)
+    : metadata
+  // copy rev over for the purposes of equality check
+  if (found) {
+    newDoc._rev = found._rev
+  }
+  if (found == null || !isEqual(newDoc, found)) {
+    return {
+      ...found,
+      ...newDoc,
+    }
+  }
+  return null
+}
+
+export async function rawUserMetadata() {
+  const db = context.getAppDB()
+  return (
+    await db.allDocs(
+      getUserMetadataParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+}
+
+export async function syncGlobalUsers() {
+  // sync user metadata
+  const db = context.getAppDB()
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(),
+    rawUserMetadata(),
+  ])
+  const toWrite = []
+  for (let user of users) {
+    const combined = await combineMetadataAndUser(user, metadata)
+    if (combined) {
+      toWrite.push(combined)
+    }
+  }
+  await db.bulkDocs(toWrite)
+}