Merge branch 'master' into grid-layout-expansion

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.30.2",
+  "version": "2.30.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/environment.ts

@@ -145,6 +145,7 @@ const environment = {
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL || "",
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
+  POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
   POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,

packages/backend-core/src/features/index.ts

@@ -7,10 +7,14 @@ import tracer from "dd-trace"
 let posthog: PostHog | undefined
 export function init(opts?: PostHogOptions) {
   if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST) {
+    console.log("initializing posthog client...")
     posthog = new PostHog(env.POSTHOG_TOKEN, {
       host: env.POSTHOG_API_HOST,
+      personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
       ...opts,
     })
+  } else {
+    console.log("posthog disabled")
   }
 }
 
@@ -128,6 +132,8 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
           continue
         }
 
+        tags[`readFromEnvironmentVars`] = true
+
         for (let feature of features) {
           let value = true
           if (feature.startsWith("!")) {
@@ -153,6 +159,8 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 
       const license = ctx?.user?.license
       if (license) {
+        tags[`readFromLicense`] = true
+
         for (const feature of license.features) {
           if (!this.isFlagName(feature)) {
             continue
@@ -175,8 +183,29 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
       }
 
       const identity = context.getIdentity()
-      if (posthog && identity?.type === IdentityType.USER) {
+      tags[`identity.type`] = identity?.type
-        const posthogFlags = await posthog.getAllFlagsAndPayloads(identity._id)
+      tags[`identity.tenantId`] = identity?.tenantId
+      tags[`identity._id`] = identity?._id
+
+      // Until we're confident this performs well, we're only enabling it in QA
+      // and test environments.
+      const usePosthog = env.isTest() || env.isQA()
+      if (usePosthog && posthog && identity?.type === IdentityType.USER) {
+        tags[`readFromPostHog`] = true
+
+        const personProperties: Record<string, string> = {}
+        if (identity.tenantId) {
+          personProperties.tenantId = identity.tenantId
+        }
+
+        const posthogFlags = await posthog.getAllFlagsAndPayloads(
+          identity._id,
+          {
+            personProperties,
+          }
+        )
+        console.log("posthog flags", JSON.stringify(posthogFlags))
+
         for (const [name, value] of Object.entries(posthogFlags.featureFlags)) {
           if (!this.isFlagName(name)) {
             // We don't want an unexpected PostHog flag to break the app, so we

packages/backend-core/src/features/tests/features.spec.ts

@@ -153,6 +153,7 @@ describe("feature flags", () => {
         mockPosthogFlags(posthogFlags)
         env.POSTHOG_TOKEN = "test"
         env.POSTHOG_API_HOST = "https://us.i.posthog.com"
+        env.POSTHOG_PERSONAL_TOKEN = "test"
       }
 
       const ctx = { user: { license: { features: licenseFlags || [] } } }
@@ -160,7 +161,11 @@ describe("feature flags", () => {
       await withEnv(env, async () => {
         // We need to pass in node-fetch here otherwise nock won't get used
         // because posthog-node uses axios under the hood.
-        init({ fetch: nodeFetch })
+        init({
+          fetch: (url, opts) => {
+            return nodeFetch(url, opts)
+          },
+        })
 
         const fullIdentity: IdentityContext = {
           _id: "us_1234",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -19,6 +19,8 @@
     helpers,
     PROTECTED_INTERNAL_COLUMNS,
     PROTECTED_EXTERNAL_COLUMNS,
+    canBeDisplayColumn,
+    canHaveDefaultColumn,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
@@ -44,6 +46,7 @@
   import { RowUtils } from "@budibase/frontend-core"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
+  import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
 
   const AUTO_TYPE = FieldType.AUTO
   const FORMULA_TYPE = FieldType.FORMULA
@@ -133,7 +136,9 @@
   }
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
-  $: required = !!editableColumn?.constraints?.presence || primaryDisplay
+  $: required = hasDefault
+    ? false
+    : !!editableColumn?.constraints?.presence || primaryDisplay
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@@ -161,15 +166,17 @@
     : availableAutoColumns
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
-    editableColumn?.type !== LINK_TYPE &&
+    canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
-    editableColumn?.type !== AUTO_TYPE &&
+  $: canHaveDefault =
-    editableColumn?.type !== JSON_TYPE &&
+    isEnabled(TENANT_FEATURE_FLAGS.DEFAULT_VALUES) &&
-    !editableColumn.autocolumn
+    canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
     editableColumn?.type !== AUTO_TYPE &&
     !editableColumn.autocolumn
+  $: hasDefault =
+    editableColumn?.default != null && editableColumn?.default !== ""
   $: externalTable = table.sourceType === DB_TYPE_EXTERNAL
   // in the case of internal tables the sourceId will just be undefined
   $: tableOptions = $tables.list.filter(
@@ -349,12 +356,15 @@
     }
   }
 
-  function onChangeRequired(e) {
+  function setRequired(req) {
-    const req = e.detail
     editableColumn.constraints.presence = req ? { allowEmpty: false } : false
     required = req
   }
 
+  function onChangeRequired(e) {
+    setRequired(e.detail)
+  }
+
   function openJsonSchemaEditor() {
     jsonSchemaModal.show()
   }
@@ -748,13 +758,37 @@
         <Toggle
           value={required}
           on:change={onChangeRequired}
-          disabled={primaryDisplay}
+          disabled={primaryDisplay || hasDefault}
           thin
           text="Required"
         />
       {/if}
     </div>
   {/if}
+
+  {#if canHaveDefault}
+    <div>
+      <ModalBindableInput
+        panel={ServerBindingPanel}
+        title="Default"
+        label="Default"
+        value={editableColumn.default}
+        on:change={e => {
+          editableColumn = {
+            ...editableColumn,
+            default: e.detail,
+          }
+
+          if (e.detail) {
+            setRequired(false)
+          }
+        }}
+        bindings={getBindings({ table })}
+        allowJS
+        context={rowGoldenSample}
+      />
+    </div>
+  {/if}
 </Layout>
 
 <div class="action-buttons">

packages/builder/src/helpers/featureFlags.js

@@ -6,6 +6,7 @@ export const TENANT_FEATURE_FLAGS = {
   USER_GROUPS: "USER_GROUPS",
   ONBOARDING_TOUR: "ONBOARDING_TOUR",
   GOOGLE_SHEETS: "GOOGLE_SHEETS",
+  DEFAULT_VALUES: "DEFAULT_VALUES",
 }
 
 export const isEnabled = featureFlag => {

packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte

@@ -16,7 +16,7 @@
 
   export let showOnboardingTypeModal
 
-  const password = Math.random().toString(36).substring(2, 22)
+  const password = generatePassword(12)
   let disabled
   let userGroups = []
 
@@ -44,7 +44,7 @@
       {
         email: "",
         role: "appUser",
-        password: Math.random().toString(36).substring(2, 22),
+        password: generatePassword(12),
         forceResetPassword: true,
         error: null,
       },
@@ -69,6 +69,14 @@
     return userData[index].error == null
   }
 
+  function generatePassword(length) {
+    const array = new Uint8Array(length)
+    window.crypto.getRandomValues(array)
+    return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+      .join("")
+      .slice(0, length)
+  }
+
   const onConfirm = () => {
     let valid = true
     userData.forEach((input, index) => {

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -216,7 +216,7 @@
       const newUser = {
         email: email,
         role: usersRole,
-        password: Math.random().toString(36).substring(2, 22),
+        password: generatePassword(12),
         forceResetPassword: true,
       }
 
@@ -288,6 +288,14 @@
     }
   }
 
+  const generatePassword = length => {
+    const array = new Uint8Array(length)
+    window.crypto.getRandomValues(array)
+    return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+      .join("")
+      .slice(0, length)
+  }
+
   onMount(async () => {
     try {
       await groups.actions.init()

packages/shared-core/src/table.ts

@@ -11,6 +11,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.AUTO]: true,
   [FieldType.INTERNAL]: true,
   [FieldType.BARCODEQR]: true,
+
   [FieldType.BIGINT]: true,
   [FieldType.BOOLEAN]: false,
   [FieldType.ARRAY]: false,
@@ -35,6 +36,30 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
   [FieldType.BIGINT]: true,
   [FieldType.BOOLEAN]: true,
   [FieldType.JSON]: true,
+
+  [FieldType.FORMULA]: false,
+  [FieldType.ATTACHMENTS]: false,
+  [FieldType.ATTACHMENT_SINGLE]: false,
+  [FieldType.SIGNATURE_SINGLE]: false,
+  [FieldType.ARRAY]: false,
+  [FieldType.LINK]: false,
+  [FieldType.BB_REFERENCE]: false,
+  [FieldType.BB_REFERENCE_SINGLE]: false,
+}
+
+const allowDefaultColumnByType: Record<FieldType, boolean> = {
+  [FieldType.NUMBER]: true,
+  [FieldType.JSON]: true,
+  [FieldType.DATETIME]: true,
+  [FieldType.LONGFORM]: true,
+  [FieldType.STRING]: true,
+
+  [FieldType.OPTIONS]: false,
+  [FieldType.AUTO]: false,
+  [FieldType.INTERNAL]: false,
+  [FieldType.BARCODEQR]: false,
+  [FieldType.BIGINT]: false,
+  [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
@@ -53,6 +78,10 @@ export function canBeSortColumn(type: FieldType): boolean {
   return !!allowSortColumnByType[type]
 }
 
+export function canHaveDefaultColumn(type: FieldType): boolean {
+  return !!allowDefaultColumnByType[type]
+}
+
 export function findDuplicateInternalColumns(table: Table): string[] {
   // maintains the case of keys
   const casedKeys = Object.keys(table.schema)

packages/types/src/documents/app/table/schema.ts

@@ -114,7 +114,6 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   type: FieldType.FORMULA
   formula: string
   formulaType?: FormulaType
-  default?: string
 }
 
 export interface BBReferenceFieldMetadata

packages/worker/src/api/controllers/global/users.ts

@@ -41,6 +41,14 @@ import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core"
 
 const MAX_USERS_UPLOAD_LIMIT = 1000
 
+const generatePassword = (length: number) => {
+  const array = new Uint8Array(length)
+  crypto.getRandomValues(array)
+  return Array.from(array, byte => byte.toString(36).padStart(2, "0"))
+    .join("")
+    .slice(0, length)
+}
+
 export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
@@ -296,7 +304,7 @@ export const onboardUsers = async (
 
   let createdPasswords: Record<string, string> = {}
   const users: User[] = ctx.request.body.map(invite => {
-    let password = Math.random().toString(36).substring(2, 22)
+    const password = generatePassword(12)
     createdPasswords[invite.email] = password
 
     return {

packages/worker/src/index.ts

@@ -96,11 +96,11 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(startupLog)
   await initPro()
   await redis.clients.init()
+  features.init()
   cache.docWritethrough.init()
   // configure events to use the pro audit log write
   // can't integrate directly into backend-core due to cyclic issues
   await events.processors.init(proSdk.auditLogs.write)
-  features.init()
 })
 
 process.on("uncaughtException", err => {