Merge remote-tracking branch 'origin/v3-ui' into feature/automation-branching-ux

packages/builder/src/components/start/CreateAppModal.svelte

@@ -118,14 +118,17 @@
       if ($values.url) {
         data.append("url", $values.url.trim())
       }
-      data.append("useTemplate", template != null)
-      if (template) {
-        data.append("templateName", template.name)
-        data.append("templateKey", template.key)
-        data.append("templateFile", $values.file)
+
+      if (template?.fromFile) {
+        data.append("useTemplate", true)
+        data.append("fileToImport", $values.file)
         if ($values.encryptionPassword?.trim()) {
           data.append("encryptionPassword", $values.encryptionPassword.trim())
         }
+      } else if (template) {
+        data.append("useTemplate", true)
+        data.append("templateName", template.name)
+        data.append("templateKey", template.key)
       }
 
       // Create App

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -14,7 +14,7 @@
   {
     "name": "Layout",
     "icon": "ClassicGridView",
-    "children": ["container", "section", "sidepanel", "modal"]
+    "children": ["container", "sidepanel", "modal"]
   },
   {
     "name": "Data",

packages/client/src/components/app/deprecated/Section.svelte

@@ -1,6 +1,6 @@
 <script>
   import { getContext } from "svelte"
-  import Placeholder from "./Placeholder.svelte"
+  import Placeholder from "../Placeholder.svelte"
 
   const { styleable, builderStore } = getContext("sdk")
   const component = getContext("component")

packages/client/src/components/app/forms/FieldGroup.svelte

@@ -1,6 +1,6 @@
 <script>
   import { getContext, setContext } from "svelte"
-  import Section from "../Section.svelte"
+  import Section from "../deprecated/Section.svelte"
 
   export let labelPosition = "above"
   export let type = "oneColumn"

packages/client/src/components/app/index.js

@@ -14,7 +14,6 @@ export { default as Placeholder } from "./Placeholder.svelte"
 
 // User facing components
 export { default as container } from "./container/Container.svelte"
-export { default as section } from "./Section.svelte"
 export { default as dataprovider } from "./DataProvider.svelte"
 export { default as divider } from "./Divider.svelte"
 export { default as screenslot } from "./ScreenSlot.svelte"
@@ -50,3 +49,4 @@ export { default as navigation } from "./deprecated/Navigation.svelte"
 export { default as cardhorizontal } from "./deprecated/CardHorizontal.svelte"
 export { default as stackedlist } from "./deprecated/StackedList.svelte"
 export { default as card } from "./deprecated/Card.svelte"
+export { default as section } from "./deprecated/Section.svelte"

packages/server/scripts/load/utils.js

@@ -29,7 +29,7 @@ exports.createApp = async apiKey => {
   const body = {
     name,
     url: `/${name}`,
-    useTemplate: "true",
+    useTemplate: true,
     templateKey: "app/school-admin-panel",
     templateName: "School Admin Panel",
   }

packages/server/src/api/controllers/application.ts

@@ -23,6 +23,7 @@ import {
   cache,
   context,
   db as dbCore,
+  docIds,
   env as envCore,
   ErrorCode,
   events,
@@ -35,7 +36,6 @@ import {
 import { USERS_TABLE_SCHEMA, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import { stringToReadStream } from "../../utilities"
 import { doesUserHaveLock } from "../../utilities/redis"
 import { cleanupAutomations } from "../../automations/utils"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
@@ -54,6 +54,11 @@ import {
   DuplicateAppResponse,
   UpdateAppRequest,
   UpdateAppResponse,
+  Database,
+  FieldType,
+  BBReferenceFieldSubType,
+  Row,
+  BBRequest,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -123,8 +128,7 @@ function checkAppName(
 }
 
 interface AppTemplate {
-  templateString?: string
-  useTemplate?: string
+  useTemplate?: boolean
   file?: {
     type?: string
     path: string
@@ -148,14 +152,7 @@ async function createInstance(appId: string, template: AppTemplate) {
   await createRoutingView()
   await createAllSearchIndex()
 
-  // replicate the template data to the instance DB
-  // this is currently very hard to test, downloading and importing template files
-  if (template && template.templateString) {
-    const { ok } = await db.load(stringToReadStream(template.templateString))
-    if (!ok) {
-      throw "Error loading database dump from memory."
-    }
-  } else if (template && template.useTemplate === "true") {
+  if (template && template.useTemplate) {
     await sdk.backups.importApp(appId, db, template)
   } else {
     // create the users table
@@ -243,14 +240,15 @@ export async function fetchAppPackage(
 
 async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const {
-    name,
-    url,
-    encryptionPassword,
-    useTemplate,
-    templateKey,
-    templateString,
-  } = ctx.request.body
+  const { body } = ctx.request
+  const { name, url, encryptionPassword, templateKey } = body
+
+  let useTemplate
+  if (typeof body.useTemplate === "string") {
+    useTemplate = body.useTemplate === "true"
+  } else if (typeof body.useTemplate === "boolean") {
+    useTemplate = body.useTemplate
+  }
 
   checkAppName(ctx, apps, name)
   const appUrl = sdk.applications.getAppUrl({ name, url })
@@ -259,16 +257,15 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const instanceConfig: AppTemplate = {
     useTemplate,
     key: templateKey,
-    templateString,
   }
-  if (ctx.request.files && ctx.request.files.templateFile) {
+  if (ctx.request.files && ctx.request.files.fileToImport) {
     instanceConfig.file = {
-      ...(ctx.request.files.templateFile as any),
+      ...(ctx.request.files.fileToImport as any),
       password: encryptionPassword,
     }
-  } else if (typeof ctx.request.body.file?.path === "string") {
+  } else if (typeof body.file?.path === "string") {
     instanceConfig.file = {
-      path: ctx.request.body.file?.path,
+      path: body.file?.path,
     }
   }
 
@@ -279,6 +276,10 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     const instance = await createInstance(appId, instanceConfig)
     const db = context.getAppDB()
 
+    if (instanceConfig.useTemplate && !instanceConfig.file) {
+      await updateUserColumns(appId, db, ctx.user._id!)
+    }
+
     const newApplication: App = {
       _id: DocumentType.APP_METADATA,
       _rev: undefined,
@@ -375,21 +376,81 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   })
 }
 
-async function creationEvents(request: any, app: App) {
+async function updateUserColumns(
+  appId: string,
+  db: Database,
+  toUserId: string
+) {
+  await context.doInAppContext(appId, async () => {
+    const allTables = await sdk.tables.getAllTables()
+    const tablesWithUserColumns = []
+    for (const table of allTables) {
+      const userColumns = Object.values(table.schema).filter(
+        f =>
+          (f.type === FieldType.BB_REFERENCE ||
+            f.type === FieldType.BB_REFERENCE_SINGLE) &&
+          f.subtype === BBReferenceFieldSubType.USER
+      )
+      if (!userColumns.length) {
+        continue
+      }
+
+      tablesWithUserColumns.push({
+        tableId: table._id!,
+        columns: userColumns.map(c => c.name),
+      })
+    }
+
+    const docsToUpdate = []
+
+    for (const { tableId, columns } of tablesWithUserColumns) {
+      const docs = await db.allDocs<Row>(
+        docIds.getRowParams(tableId, null, { include_docs: true })
+      )
+      const rows = docs.rows.map(d => d.doc!)
+
+      for (const row of rows) {
+        let shouldUpdate = false
+        const updatedColumns = columns.reduce<Row>((newColumns, column) => {
+          if (row[column]) {
+            shouldUpdate = true
+            if (Array.isArray(row[column])) {
+              newColumns[column] = row[column]?.map(() => toUserId)
+            } else if (row[column]) {
+              newColumns[column] = toUserId
+            }
+          }
+          return newColumns
+        }, {})
+
+        if (shouldUpdate) {
+          docsToUpdate.push({
+            ...row,
+            ...updatedColumns,
+          })
+        }
+      }
+    }
+
+    await db.bulkDocs(docsToUpdate)
+  })
+}
+
+async function creationEvents(request: BBRequest<CreateAppRequest>, app: App) {
   let creationFns: ((app: App) => Promise<void>)[] = []
 
-  const body = request.body
-  if (body.useTemplate === "true") {
+  const { useTemplate, templateKey, file } = request.body
+  if (useTemplate === "true") {
     // from template
-    if (body.templateKey && body.templateKey !== "undefined") {
-      creationFns.push(a => events.app.templateImported(a, body.templateKey))
+    if (templateKey && templateKey !== "undefined") {
+      creationFns.push(a => events.app.templateImported(a, templateKey))
     }
     // from file
-    else if (request.files?.templateFile) {
+    else if (request.files?.fileToImport) {
       creationFns.push(a => events.app.fileImported(a))
     }
     // from server file path
-    else if (request.body.file) {
+    else if (file) {
       // explicitly pass in the newly created app id
       creationFns.push(a => events.app.duplicated(a, app.appId))
     }
@@ -399,16 +460,14 @@ async function creationEvents(request: any, app: App) {
     }
   }
 
-  if (!request.duplicate) {
-    creationFns.push(a => events.app.created(a))
-  }
+  creationFns.push(a => events.app.created(a))
 
   for (let fn of creationFns) {
     await fn(app)
   }
 }
 
-async function appPostCreate(ctx: UserCtx, app: App) {
+async function appPostCreate(ctx: UserCtx<CreateAppRequest, App>, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -419,7 +478,7 @@ async function appPostCreate(ctx: UserCtx, app: App) {
   await creationEvents(ctx.request, app)
 
   // app import, template creation and duplication
-  if (ctx.request.body.useTemplate === "true") {
+  if (ctx.request.body.useTemplate) {
     const { rows } = await getUniqueRows([app.appId])
     const rowCount = rows ? rows.length : 0
     if (rowCount) {

packages/server/src/api/routes/tests/application.spec.ts

@@ -21,6 +21,7 @@ import tk from "timekeeper"
 import * as uuid from "uuid"
 import { structures } from "@budibase/backend-core/tests"
 import nock from "nock"
+import path from "path"
 
 describe("/applications", () => {
   let config = setup.getConfig()
@@ -137,11 +138,17 @@ describe("/applications", () => {
     })
 
     it("creates app from template", async () => {
+      nock("https://prod-budi-templates.s3-eu-west-1.amazonaws.com")
+        .get(`/templates/app/agency-client-portal.tar.gz`)
+        .replyWithFile(
+          200,
+          path.resolve(__dirname, "data", "agency-client-portal.tar.gz")
+        )
+
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateKey: "test",
-        templateString: "{}",
+        templateKey: "app/agency-client-portal",
       })
       expect(app._id).toBeDefined()
       expect(events.app.created).toHaveBeenCalledTimes(1)
@@ -152,7 +159,7 @@ describe("/applications", () => {
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateFile: "src/api/routes/tests/data/export.txt",
+        fileToImport: "src/api/routes/tests/data/export.txt",
       })
       expect(app._id).toBeDefined()
       expect(events.app.created).toHaveBeenCalledTimes(1)
@@ -172,7 +179,7 @@ describe("/applications", () => {
       const app = await config.api.application.create({
         name: utils.newid(),
         useTemplate: "true",
-        templateFile: "src/api/routes/tests/data/old-app.txt",
+        fileToImport: "src/api/routes/tests/data/old-app.txt",
       })
       expect(app._id).toBeDefined()
       expect(app.navigation).toBeDefined()

packages/server/src/api/routes/utils/validators.ts

@@ -357,9 +357,7 @@ export function applicationValidator(opts = { isCreate: true }) {
     _id: OPTIONAL_STRING,
     _rev: OPTIONAL_STRING,
     url: OPTIONAL_STRING,
-    template: Joi.object({
-      templateString: OPTIONAL_STRING,
-    }),
+    template: Joi.object({}),
   }
 
   const appNameValidator = Joi.string()
@@ -392,9 +390,7 @@ export function applicationValidator(opts = { isCreate: true }) {
       _rev: OPTIONAL_STRING,
       name: appNameValidator,
       url: OPTIONAL_STRING,
-      template: Joi.object({
-        templateString: OPTIONAL_STRING,
-      }).unknown(true),
+      template: Joi.object({}).unknown(true),
       snippets: snippetValidator,
     }).unknown(true)
   )

packages/server/src/tests/utilities/api/application.ts

@@ -17,8 +17,8 @@ export class ApplicationAPI extends TestAPI {
     app: CreateAppRequest,
     expectations?: Expectations
   ): Promise<App> => {
-    const files = app.templateFile ? { templateFile: app.templateFile } : {}
-    delete app.templateFile
+    const files = app.fileToImport ? { fileToImport: app.fileToImport } : {}
+    delete app.fileToImport
     return await this._post<App>("/api/applications", {
       fields: app,
       files,

packages/server/src/utilities/index.ts

@@ -2,9 +2,6 @@ import env from "../environment"
 import { context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"
 import { Document } from "@budibase/types"
-import stream from "stream"
-
-const Readable = stream.Readable
 
 export function wait(ms: number) {
   return new Promise(resolve => setTimeout(resolve, ms))
@@ -98,15 +95,6 @@ export function escapeDangerousCharacters(string: string) {
     .replace(/[\t]/g, "\\t")
 }
 
-export function stringToReadStream(string: string) {
-  return new Readable({
-    read() {
-      this.push(string)
-      this.push(null)
-    },
-  })
-}
-
 export function formatBytes(bytes: string) {
   const units = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]
   const byteIncrements = 1024

packages/types/src/api/web/application.ts

@@ -7,10 +7,8 @@ export interface CreateAppRequest {
   useTemplate?: string
   templateName?: string
   templateKey?: string
-  templateFile?: string
-  includeSampleData?: boolean
+  fileToImport?: string
   encryptionPassword?: string
-  templateString?: string
   file?: { path: string }
 }