From aaaf1732954fdd3dee5ade7a255bed4b88c02d50 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 17 Jul 2024 11:02:47 +0100 Subject: [PATCH 1/8] Initial UI. --- .../DataTable/modals/CreateEditColumn.svelte | 45 +++++++++++++++++-- .../types/src/documents/app/table/schema.ts | 1 - 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index a9ea90242a..6f657f57d6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -133,7 +133,9 @@ } $: initialiseField(field, savingColumn) $: checkConstraints(editableColumn) - $: required = !!editableColumn?.constraints?.presence || primaryDisplay + $: required = hasDefault + ? false + : !!editableColumn?.constraints?.presence || primaryDisplay $: uneditable = $tables.selected?._id === TableNames.USERS && UNEDITABLE_USER_FIELDS.includes(editableColumn.name) @@ -165,11 +167,19 @@ editableColumn?.type !== AUTO_TYPE && editableColumn?.type !== JSON_TYPE && !editableColumn.autocolumn + $: canHaveDefault = + editableColumn?.type === FieldType.NUMBER || + editableColumn?.type === FieldType.JSON || + editableColumn?.type === FieldType.DATETIME || + editableColumn?.type === FieldType.LONGFORM || + editableColumn?.type === FieldType.STRING $: canBeRequired = editableColumn?.type !== LINK_TYPE && !uneditable && editableColumn?.type !== AUTO_TYPE && !editableColumn.autocolumn + $: hasDefault = + editableColumn?.default != null && editableColumn?.default !== "" $: externalTable = table.sourceType === DB_TYPE_EXTERNAL // in the case of internal tables the sourceId will just be undefined $: tableOptions = $tables.list.filter( @@ -349,12 +359,15 @@ } } - function onChangeRequired(e) { - const req = e.detail + function setRequired(req) { editableColumn.constraints.presence = req ? { allowEmpty: false } : false required = req } + function onChangeRequired(e) { + setRequired(e.detail) + } + function openJsonSchemaEditor() { jsonSchemaModal.show() } @@ -748,13 +761,37 @@ {/if} {/if} + + {#if canHaveDefault} +
+ { + editableColumn = { + ...editableColumn, + default: e.detail, + } + + if (e.detail) { + setRequired(false) + } + }} + bindings={getBindings({ table })} + allowJS + context={rowGoldenSample} + /> +
+ {/if}
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 7b89b610be..6078f73d1d 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -114,7 +114,6 @@ export interface FormulaFieldMetadata extends BaseFieldSchema { type: FieldType.FORMULA formula: string formulaType?: FormulaType - default?: string } export interface BBReferenceFieldMetadata From 1b556f29fe07ea6413ed82377aed98b5e6469743 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 9 Aug 2024 14:00:16 +0100 Subject: [PATCH 2/8] Rebase on posthog-feature-flags --- .../backend/DataTable/modals/CreateEditColumn.svelte | 12 +++++++----- packages/builder/src/helpers/featureFlags.js | 1 + packages/worker/src/index.ts | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 6f657f57d6..06c47f94af 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -44,6 +44,7 @@ import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" import OptionsEditor from "./OptionsEditor.svelte" + import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags" const AUTO_TYPE = FieldType.AUTO const FORMULA_TYPE = FieldType.FORMULA @@ -168,11 +169,12 @@ editableColumn?.type !== JSON_TYPE && !editableColumn.autocolumn $: canHaveDefault = - editableColumn?.type === FieldType.NUMBER || - editableColumn?.type === FieldType.JSON || - editableColumn?.type === FieldType.DATETIME || - editableColumn?.type === FieldType.LONGFORM || - editableColumn?.type === FieldType.STRING + isEnabled(TENANT_FEATURE_FLAGS.DEFAULT_VALUES) && + (editableColumn?.type === FieldType.NUMBER || + editableColumn?.type === FieldType.JSON || + editableColumn?.type === FieldType.DATETIME || + editableColumn?.type === FieldType.LONGFORM || + editableColumn?.type === FieldType.STRING) $: canBeRequired = editableColumn?.type !== LINK_TYPE && !uneditable && diff --git a/packages/builder/src/helpers/featureFlags.js b/packages/builder/src/helpers/featureFlags.js index fe30fb9980..824e3c3f3e 100644 --- a/packages/builder/src/helpers/featureFlags.js +++ b/packages/builder/src/helpers/featureFlags.js @@ -6,6 +6,7 @@ export const TENANT_FEATURE_FLAGS = { USER_GROUPS: "USER_GROUPS", ONBOARDING_TOUR: "ONBOARDING_TOUR", GOOGLE_SHEETS: "GOOGLE_SHEETS", + DEFAULT_VALUES: "DEFAULT_VALUES", } export const isEnabled = featureFlag => { diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index d59d8d96ef..79e6f4493d 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -96,11 +96,11 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => { console.log(startupLog) await initPro() await redis.clients.init() + features.init() cache.docWritethrough.init() // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues await events.processors.init(proSdk.auditLogs.write) - features.init() }) process.on("uncaughtException", err => { From 700356a1ef204b8771d10ca73589b140d47e0e26 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 Aug 2024 11:08:58 +0100 Subject: [PATCH 3/8] Factor out default value check to shared-core. --- .../DataTable/modals/CreateEditColumn.svelte | 13 +++------ packages/shared-core/src/table.ts | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 06c47f94af..98243c6e2f 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -19,6 +19,8 @@ helpers, PROTECTED_INTERNAL_COLUMNS, PROTECTED_EXTERNAL_COLUMNS, + canBeDisplayColumn, + canHaveDefaultColumn, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" @@ -164,17 +166,10 @@ : availableAutoColumns // used to select what different options can be displayed for column type $: canBeDisplay = - editableColumn?.type !== LINK_TYPE && - editableColumn?.type !== AUTO_TYPE && - editableColumn?.type !== JSON_TYPE && - !editableColumn.autocolumn + canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn $: canHaveDefault = isEnabled(TENANT_FEATURE_FLAGS.DEFAULT_VALUES) && - (editableColumn?.type === FieldType.NUMBER || - editableColumn?.type === FieldType.JSON || - editableColumn?.type === FieldType.DATETIME || - editableColumn?.type === FieldType.LONGFORM || - editableColumn?.type === FieldType.STRING) + canHaveDefaultColumn(editableColumn.type) $: canBeRequired = editableColumn?.type !== LINK_TYPE && !uneditable && diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 9e7626cb1c..8a8069ce4d 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -11,6 +11,7 @@ const allowDisplayColumnByType: Record = { [FieldType.AUTO]: true, [FieldType.INTERNAL]: true, [FieldType.BARCODEQR]: true, + [FieldType.BIGINT]: true, [FieldType.BOOLEAN]: false, [FieldType.ARRAY]: false, @@ -35,6 +36,30 @@ const allowSortColumnByType: Record = { [FieldType.BIGINT]: true, [FieldType.BOOLEAN]: true, [FieldType.JSON]: true, + + [FieldType.FORMULA]: false, + [FieldType.ATTACHMENTS]: false, + [FieldType.ATTACHMENT_SINGLE]: false, + [FieldType.SIGNATURE_SINGLE]: false, + [FieldType.ARRAY]: false, + [FieldType.LINK]: false, + [FieldType.BB_REFERENCE]: false, + [FieldType.BB_REFERENCE_SINGLE]: false, +} + +const allowDefaultColumnByType: Record = { + [FieldType.NUMBER]: true, + [FieldType.JSON]: true, + [FieldType.DATETIME]: true, + [FieldType.LONGFORM]: true, + [FieldType.STRING]: true, + + [FieldType.OPTIONS]: false, + [FieldType.AUTO]: false, + [FieldType.INTERNAL]: false, + [FieldType.BARCODEQR]: false, + [FieldType.BIGINT]: false, + [FieldType.BOOLEAN]: false, [FieldType.FORMULA]: false, [FieldType.ATTACHMENTS]: false, [FieldType.ATTACHMENT_SINGLE]: false, @@ -53,6 +78,10 @@ export function canBeSortColumn(type: FieldType): boolean { return !!allowSortColumnByType[type] } +export function canHaveDefaultColumn(type: FieldType): boolean { + return !!allowDefaultColumnByType[type] +} + export function findDuplicateInternalColumns(table: Table): string[] { // maintains the case of keys const casedKeys = Object.keys(table.schema) From 72a37ed1a70a528851fea241002180b9cd127444 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 Aug 2024 15:33:48 +0100 Subject: [PATCH 4/8] Add some logging to see if I can figure out why PostHog flags aren't coming through. --- packages/backend-core/src/features/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 25c9b260d8..af8bd00efc 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -7,10 +7,13 @@ import tracer from "dd-trace" let posthog: PostHog | undefined export function init(opts?: PostHogOptions) { if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST) { + console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, ...opts, }) + } else { + console.log("posthog disabled") } } @@ -128,6 +131,8 @@ export class FlagSet, T extends { [key: string]: V }> { continue } + tags[`readFromEnvironmentVars`] = true + for (let feature of features) { let value = true if (feature.startsWith("!")) { @@ -153,6 +158,8 @@ export class FlagSet, T extends { [key: string]: V }> { const license = ctx?.user?.license if (license) { + tags[`readFromLicense`] = true + for (const feature of license.features) { if (!this.isFlagName(feature)) { continue @@ -175,7 +182,13 @@ export class FlagSet, T extends { [key: string]: V }> { } const identity = context.getIdentity() + tags[`identity.type`] = identity?.type + tags[`identity.tenantId`] = identity?.tenantId + tags[`identity._id`] = identity?._id + if (posthog && identity?.type === IdentityType.USER) { + tags[`readFromPostHog`] = true + const posthogFlags = await posthog.getAllFlagsAndPayloads(identity._id) for (const [name, value] of Object.entries(posthogFlags.featureFlags)) { if (!this.isFlagName(name)) { From b01c111567504e476470e375e13c566c5d26d6f1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 Aug 2024 15:34:23 +0100 Subject: [PATCH 5/8] Log PostHog payload. --- packages/backend-core/src/features/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index af8bd00efc..d3f00af87d 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -190,6 +190,8 @@ export class FlagSet, T extends { [key: string]: V }> { tags[`readFromPostHog`] = true const posthogFlags = await posthog.getAllFlagsAndPayloads(identity._id) + console.log("posthog flags", JSON.stringify(posthogFlags)) + for (const [name, value] of Object.entries(posthogFlags.featureFlags)) { if (!this.isFlagName(name)) { // We don't want an unexpected PostHog flag to break the app, so we From b5465f1b6367510de1ba2f8a2506facc367ad32e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 12 Aug 2024 16:32:25 +0100 Subject: [PATCH 6/8] Disable PostHog feature flags in prod. --- packages/backend-core/src/environment.ts | 1 + packages/backend-core/src/features/index.ts | 18 ++++++++++++++++-- .../src/features/tests/features.spec.ts | 7 ++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index e064398153..6377d682d0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -145,6 +145,7 @@ const environment = { COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, PLATFORM_URL: process.env.PLATFORM_URL || "", POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, + POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN, POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com", ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index d3f00af87d..5305da4764 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -10,6 +10,7 @@ export function init(opts?: PostHogOptions) { console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, + personalApiKey: env.POSTHOG_PERSONAL_TOKEN, ...opts, }) } else { @@ -186,10 +187,23 @@ export class FlagSet, T extends { [key: string]: V }> { tags[`identity.tenantId`] = identity?.tenantId tags[`identity._id`] = identity?._id - if (posthog && identity?.type === IdentityType.USER) { + // Until we're confident this performs well, we're only enabling it in QA + // and test environments. + const usePosthog = env.isTest() || env.isQA() + if (usePosthog && posthog && identity?.type === IdentityType.USER) { tags[`readFromPostHog`] = true - const posthogFlags = await posthog.getAllFlagsAndPayloads(identity._id) + const personProperties: Record = {} + if (identity.tenantId) { + personProperties.tenantId = identity.tenantId + } + + const posthogFlags = await posthog.getAllFlagsAndPayloads( + identity._id, + { + personProperties, + } + ) console.log("posthog flags", JSON.stringify(posthogFlags)) for (const [name, value] of Object.entries(posthogFlags.featureFlags)) { diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9c928f5545..1b6a6e041b 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -153,6 +153,7 @@ describe("feature flags", () => { mockPosthogFlags(posthogFlags) env.POSTHOG_TOKEN = "test" env.POSTHOG_API_HOST = "https://us.i.posthog.com" + env.POSTHOG_PERSONAL_TOKEN = "test" } const ctx = { user: { license: { features: licenseFlags || [] } } } @@ -160,7 +161,11 @@ describe("feature flags", () => { await withEnv(env, async () => { // We need to pass in node-fetch here otherwise nock won't get used // because posthog-node uses axios under the hood. - init({ fetch: nodeFetch }) + init({ + fetch: (url, opts) => { + return nodeFetch(url, opts) + }, + }) const fullIdentity: IdentityContext = { _id: "us_1234", From 151fff51c530684ce7d840660e0aa3efbb931d27 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Mon, 12 Aug 2024 21:37:59 +0100 Subject: [PATCH 7/8] Make generated passwords longer (#14362) * Make generated passwords longer * Use crypto for generating passwords * Remove comments * Generate password with length 12 --- .../users/users/_components/AddUserModal.svelte | 12 ++++++++++-- .../pages/builder/portal/users/users/index.svelte | 10 +++++++++- packages/worker/src/api/controllers/global/users.ts | 10 +++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte index 5077239882..36709ae9c0 100644 --- a/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/_components/AddUserModal.svelte @@ -16,7 +16,7 @@ export let showOnboardingTypeModal - const password = Math.random().toString(36).substring(2, 22) + const password = generatePassword(12) let disabled let userGroups = [] @@ -44,7 +44,7 @@ { email: "", role: "appUser", - password: Math.random().toString(36).substring(2, 22), + password: generatePassword(12), forceResetPassword: true, error: null, }, @@ -69,6 +69,14 @@ return userData[index].error == null } + function generatePassword(length) { + const array = new Uint8Array(length) + window.crypto.getRandomValues(array) + return Array.from(array, byte => byte.toString(36).padStart(2, "0")) + .join("") + .slice(0, length) + } + const onConfirm = () => { let valid = true userData.forEach((input, index) => { diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index 58da310104..01d23afa67 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -216,7 +216,7 @@ const newUser = { email: email, role: usersRole, - password: Math.random().toString(36).substring(2, 22), + password: generatePassword(12), forceResetPassword: true, } @@ -288,6 +288,14 @@ } } + const generatePassword = length => { + const array = new Uint8Array(length) + window.crypto.getRandomValues(array) + return Array.from(array, byte => byte.toString(36).padStart(2, "0")) + .join("") + .slice(0, length) + } + onMount(async () => { try { await groups.actions.init() diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 273eec279c..b039376d9b 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -41,6 +41,14 @@ import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core" const MAX_USERS_UPLOAD_LIMIT = 1000 +const generatePassword = (length: number) => { + const array = new Uint8Array(length) + crypto.getRandomValues(array) + return Array.from(array, byte => byte.toString(36).padStart(2, "0")) + .join("") + .slice(0, length) +} + export const save = async (ctx: UserCtx) => { try { const currentUserId = ctx.user?._id @@ -296,7 +304,7 @@ export const onboardUsers = async ( let createdPasswords: Record = {} const users: User[] = ctx.request.body.map(invite => { - let password = Math.random().toString(36).substring(2, 22) + const password = generatePassword(12) createdPasswords[invite.email] = password return { From 7c3593c9291aa2bf87f475a029844d4c8c2b964a Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 13 Aug 2024 08:15:59 +0000 Subject: [PATCH 8/8] Bump version to 2.30.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c9b38f1031..070d726a42 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.30.2", + "version": "2.30.3", "npmClient": "yarn", "packages": [ "packages/*",