Merge branch 'master' into chore/refactor-finalizerow

.github/workflows/tag-release.yml

@@ -13,7 +13,6 @@ on:
         options:
           - patch
           - minor
-          - major
         required: true
 
 jobs:

hosting/single/runner.sh

@@ -81,11 +81,17 @@ mkdir -p ${DATA_DIR}/minio
 mkdir -p ${DATA_DIR}/redis
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 
-sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
+REDIS_CONFIG="/etc/redis/redis.conf"
+sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}"
+
+if [[ -n "${USE_DEFAULT_REDIS_CONFIG}" ]]; then
+  REDIS_CONFIG=""
+fi
+
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+  redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 else
-  redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
+  redis-server "${REDIS_CONFIG}" > /dev/stdout 2>&1 &
 fi
 /bbcouch-runner.sh &
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.0.1",
+  "version": "3.0.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/features/features.ts

@@ -272,7 +272,6 @@ export const flags = new FlagSet({
   [FeatureFlag.SQS]: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),
   [FeatureFlag.BUDIBASE_AI]: Flag.boolean(env.isDev()),
 })
 

packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte

@@ -9,7 +9,7 @@
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { flags } from "stores/builder"
-  import { featureFlags } from "stores/portal"
+  import { featureFlags, licensing } from "stores/portal"
   import { API } from "api"
   import MagicWand from "../../../../assets/MagicWand.svelte"
 
@@ -26,7 +26,9 @@
   let aiCronPrompt = ""
   let loadingAICronExpression = false
 
-  $: aiEnabled = $featureFlags.AI_CUSTOM_CONFIGS || $featureFlags.BUDIBASE_AI
+  $: aiEnabled =
+    ($featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled) ||
+    ($featureFlags.BUDIBASE_AI && $licensing.budibaseAIEnabled)
   $: {
     if (cronExpression) {
       try {

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -65,7 +65,7 @@
   let tableOptions
   let errorChecker = new RelationshipErrorChecker(
     invalidThroughTable,
-    relationshipExists
+    manyToManyRelationshipExistsFn
   )
   let errors = {}
   let fromPrimary, fromForeign, fromColumn, toColumn
@@ -125,7 +125,7 @@
     }
     return false
   }
-  function relationshipExists() {
+  function manyToManyRelationshipExistsFn() {
     if (
       originalFromTable &&
       originalToTable &&
@@ -141,16 +141,14 @@
       datasource.entities[getTable(toId).name].schema
     ).filter(value => value.through)
 
-    const matchAgainstUserInput = (fromTableId, toTableId) =>
+    const matchAgainstUserInput = link =>
-      (fromTableId === fromId && toTableId === toId) ||
+      (link.throughTo === throughToKey &&
-      (fromTableId === toId && toTableId === fromId)
+        link.throughFrom === throughFromKey) ||
+      (link.throughTo === throughFromKey && link.throughFrom === throughToKey)
 
-    return !!fromThroughLinks.find(from =>
+    const allLinks = [...fromThroughLinks, ...toThroughLinks]
-      toThroughLinks.find(
+    return !!allLinks.find(
-        to =>
+      link => link.through === throughId && matchAgainstUserInput(link)
-          from.through === to.through &&
-          matchAgainstUserInput(from.tableId, to.tableId)
-      )
     )
   }
 
@@ -181,16 +179,15 @@
       relationshipType: errorChecker.relationshipTypeSet(relationshipType),
       fromTable:
         errorChecker.tableSet(fromTable) ||
-        errorChecker.doesRelationshipExists() ||
         errorChecker.differentTables(fromId, toId, throughId),
       toTable:
         errorChecker.tableSet(toTable) ||
-        errorChecker.doesRelationshipExists() ||
         errorChecker.differentTables(toId, fromId, throughId),
       throughTable:
         errorChecker.throughTableSet(throughTable) ||
         errorChecker.throughIsNullable() ||
-        errorChecker.differentTables(throughId, fromId, toId),
+        errorChecker.differentTables(throughId, fromId, toId) ||
+        errorChecker.doesRelationshipExists(),
       throughFromKey:
         errorChecker.manyForeignKeySet(throughFromKey) ||
         errorChecker.manyTypeMismatch(
@@ -198,7 +195,8 @@
           throughTable,
           fromTable.primary[0],
           throughToKey
-        ),
+        ) ||
+        errorChecker.differentColumns(throughFromKey, throughToKey),
       throughToKey:
         errorChecker.manyForeignKeySet(throughToKey) ||
         errorChecker.manyTypeMismatch(
@@ -372,6 +370,16 @@
       fromColumn = selectedFromTable.name
       fromPrimary = selectedFromTable?.primary[0] || null
     }
+    if (relationshipType === RelationshipType.MANY_TO_MANY) {
+      relationshipPart1 = PrettyRelationshipDefinitions.MANY
+      relationshipPart2 = PrettyRelationshipDefinitions.MANY
+    } else if (relationshipType === RelationshipType.MANY_TO_ONE) {
+      relationshipPart1 = PrettyRelationshipDefinitions.ONE
+      relationshipPart2 = PrettyRelationshipDefinitions.MANY
+    } else {
+      relationshipPart1 = PrettyRelationshipDefinitions.MANY
+      relationshipPart2 = PrettyRelationshipDefinitions.ONE
+    }
   })
 </script>
 

packages/builder/src/components/backend/Datasources/relationshipErrors.js

@@ -3,6 +3,7 @@ import { RelationshipType } from "@budibase/types"
 const typeMismatch = "Column type of the foreign key must match the primary key"
 const columnBeingUsed = "Column name cannot be an existing column"
 const mustBeDifferentTables = "From/to/through tables must be different"
+const mustBeDifferentColumns = "Foreign keys must be different"
 const primaryKeyNotSet = "Please pick the primary key"
 const throughNotNullable =
   "Ensure non-key columns are nullable or auto-generated"
@@ -30,9 +31,9 @@ function typeMismatchCheck(fromTable, toTable, primary, foreign) {
 }
 
 export class RelationshipErrorChecker {
-  constructor(invalidThroughTableFn, relationshipExistsFn) {
+  constructor(invalidThroughTableFn, manyToManyRelationshipExistsFn) {
     this.invalidThroughTable = invalidThroughTableFn
-    this.relationshipExists = relationshipExistsFn
+    this.manyToManyRelationshipExists = manyToManyRelationshipExistsFn
   }
 
   setType(type) {
@@ -72,7 +73,7 @@ export class RelationshipErrorChecker {
   }
 
   doesRelationshipExists() {
-    return this.isMany() && this.relationshipExists()
+    return this.isMany() && this.manyToManyRelationshipExists()
       ? relationshipAlreadyExists
       : null
   }
@@ -83,6 +84,11 @@ export class RelationshipErrorChecker {
     return error ? mustBeDifferentTables : null
   }
 
+  differentColumns(columnA, columnB) {
+    const error = columnA && columnB && columnA === columnB
+    return error ? mustBeDifferentColumns : null
+  }
+
   columnBeingUsed(table, column, ogName) {
     return isColumnNameBeingUsed(table, column, ogName) ? columnBeingUsed : null
   }

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -5,13 +5,13 @@
     Button,
     Drawer,
     DrawerContent,
-    Helpers,
   } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import FilterBuilder from "./FilterBuilder.svelte"
   import { tables, selectedScreen } from "stores/builder"
   import { search } from "@budibase/frontend-core"
+  import { utils } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
 
@@ -22,7 +22,7 @@
 
   let drawer
 
-  $: localFilters = Helpers.cloneDeep(value)
+  $: localFilters = value
   $: datasource = getDatasourceForProvider($selectedScreen, componentInstance)
   $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema
   $: schemaFields = search.getFields(
@@ -30,8 +30,7 @@
     Object.values(schema || dsSchema || {}),
     { allowLinks: true }
   )
-
+  $: text = getText(value)
-  $: text = getText(value?.groups)
 
   async function saveFilter() {
     dispatch("change", localFilters)
@@ -39,11 +38,14 @@
     drawer.hide()
   }
 
-  const getText = (filterGroups = []) => {
+  const getText = filters => {
-    const allFilters = filterGroups.reduce((acc, group) => {
+    if (Array.isArray(filters)) {
+      filters = utils.processSearchFilters(filters)
+    }
+    const groups = filters?.groups || []
+    const allFilters = groups.reduce((acc, group) => {
       return (acc += group.filters.filter(filter => filter.field).length)
     }, 0)
-
     if (allFilters === 0) {
       return "No filters set"
     } else {
@@ -62,7 +64,7 @@
   on:drawerShow
   on:drawerShow={() => {
     // Reset to the currently available value.
-    localFilters = Helpers.cloneDeep(value)
+    localFilters = value
   }}
 >
   <Button cta slot="buttons" on:click={saveFilter}>Save</Button>

packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js

@@ -1,7 +1,7 @@
 import { it, expect, describe, vi } from "vitest"
 import AISettings from "./index.svelte"
 import { render, fireEvent } from "@testing-library/svelte"
-import { admin, licensing } from "stores/portal"
+import { admin, licensing, featureFlags } from "stores/portal"
 import { notifications } from "@budibase/bbui"
 
 vi.spyOn(notifications, "error").mockImplementation(vi.fn)
@@ -12,12 +12,17 @@ const Hosting = {
   Self: "self",
 }
 
-function setupEnv(hosting, features = {}) {
+function setupEnv(hosting, features = {}, flags = {}) {
   const defaultFeatures = {
     budibaseAIEnabled: false,
     customAIConfigsEnabled: false,
     ...features,
   }
+  const defaultFlags = {
+    BUDIBASE_AI: false,
+    AI_CUSTOM_CONFIGS: false,
+    ...flags,
+  }
   admin.subscribe = vi.fn().mockImplementation(callback => {
     callback({ cloud: hosting === Hosting.Cloud })
     return () => {}
@@ -26,6 +31,10 @@ function setupEnv(hosting, features = {}) {
     callback(defaultFeatures)
     return () => {}
   })
+  featureFlags.subscribe = vi.fn().mockImplementation(callback => {
+    callback(defaultFlags)
+    return () => {}
+  })
 }
 
 describe("AISettings", () => {
@@ -72,7 +81,11 @@ describe("AISettings", () => {
       let addConfigurationButton
       let configModal
 
-      setupEnv(Hosting.Cloud, { customAIConfigsEnabled: true })
+      setupEnv(
+        Hosting.Cloud,
+        { customAIConfigsEnabled: true },
+        { AI_CUSTOM_CONFIGS: true }
+      )
       instance = render(AISettings)
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).toBeInTheDocument()
@@ -85,7 +98,11 @@ describe("AISettings", () => {
       let addConfigurationButton
       let configModal
 
-      setupEnv(Hosting.Self, { customAIConfigsEnabled: true })
+      setupEnv(
+        Hosting.Self,
+        { customAIConfigsEnabled: true },
+        { AI_CUSTOM_CONFIGS: true }
+      )
       instance = render(AISettings)
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).toBeInTheDocument()

packages/builder/src/pages/builder/portal/settings/ai/index.svelte

@@ -12,7 +12,7 @@
     Tags,
     Tag,
   } from "@budibase/bbui"
-  import { admin, licensing } from "stores/portal"
+  import { admin, licensing, featureFlags } from "stores/portal"
   import { API } from "api"
   import AIConfigModal from "./ConfigModal.svelte"
   import AIConfigTile from "./AIConfigTile.svelte"
@@ -27,7 +27,8 @@
   let editingUuid
 
   $: isCloud = $admin.cloud
-  $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled
+  $: customAIConfigsEnabled =
+    $featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled
 
   async function fetchAIConfig() {
     try {

packages/builder/src/pages/builder/portal/settings/index.svelte

@@ -1,8 +1,8 @@
 <script>
   import { redirect } from "@roxi/routify"
-  import { licensing } from "stores/portal"
+  import { licensing, featureFlags } from "stores/portal"
 
-  if ($licensing.customAIConfigsEnabled) {
+  if ($featureFlags.AI_CUSTOM_CONFIGS && $licensing.customAIConfigsEnabled) {
     $redirect("./ai")
   } else {
     $redirect("./auth")

packages/server/src/api/controllers/row/staticFormula.ts

@@ -4,8 +4,15 @@ import {
   processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context } from "@budibase/backend-core"
+import { context, features } from "@budibase/backend-core"
-import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
+import {
+  Table,
+  Row,
+  FeatureFlag,
+  FormulaType,
+  FieldType,
+  ViewV2,
+} from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
@@ -145,8 +152,10 @@ export async function finaliseRow(
     contextRows: [enrichedRow],
   })
   const aiEnabled =
-    (await pro.features.isBudibaseAIEnabled()) ||
+    ((await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
-    (await pro.features.isAICustomConfigsEnabled())
+      (await pro.features.isBudibaseAIEnabled())) ||
+    ((await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+      (await pro.features.isAICustomConfigsEnabled()))
   if (aiEnabled) {
     row = await processAIColumns(table, row, {
       contextRows: [enrichedRow],
@@ -160,7 +169,7 @@ export async function finaliseRow(
     dynamic: false,
   })
   if (aiEnabled) {
-    enrichedRow = await processAIColumns(table, row, {
+    enrichedRow = await processAIColumns(table, enrichedRow, {
       contextRows: [enrichedRow],
     })
   }

packages/server/src/automations/steps/openai.ts

@@ -7,17 +7,17 @@ import {
   AutomationIOType,
   OpenAIStepInputs,
   OpenAIStepOutputs,
+  FeatureFlag,
 } from "@budibase/types"
-import { env } from "@budibase/backend-core"
+import { env, features } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
 enum Model {
-  GPT_35_TURBO = "gpt-3.5-turbo",
-  // will only work with api keys that have access to the GPT4 API
-  GPT_4 = "gpt-4",
-  GPT_4O = "gpt-4o",
   GPT_4O_MINI = "gpt-4o-mini",
+  GPT_4O = "gpt-4o",
+  GPT_4 = "gpt-4",
+  GPT_35_TURBO = "gpt-3.5-turbo",
 }
 
 export const definition: AutomationStepDefinition = {
@@ -99,8 +99,12 @@ export async function run({
 
   try {
     let response
-    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
+    const customConfigsEnabled =
-    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
+      (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+      (await pro.features.isAICustomConfigsEnabled())
+    const budibaseAIEnabled =
+      (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+      (await pro.features.isBudibaseAIEnabled())
 
     if (budibaseAIEnabled || customConfigsEnabled) {
       const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model)

packages/server/src/sdk/app/tables/create.ts

@@ -1,10 +1,10 @@
-import { FeatureFlag, Row, Table } from "@budibase/types"
+import { Row, Table } from "@budibase/types"
 
 import * as external from "./external"
 import * as internal from "./internal"
 import { isExternal } from "./utils"
 import { setPermissions } from "../permissions"
-import { features, roles } from "@budibase/backend-core"
+import { roles } from "@budibase/backend-core"
 
 export async function create(
   table: Omit<Table, "_id" | "_rev">,
@@ -18,16 +18,10 @@ export async function create(
     createdTable = await internal.create(table, rows, userId)
   }
 
-  const setExplicitPermission = await features.flags.isEnabled(
+  await setPermissions(createdTable._id!, {
-    FeatureFlag.TABLES_DEFAULT_ADMIN
+    writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
-  )
+    readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
-
+  })
-  if (setExplicitPermission) {
-    await setPermissions(createdTable._id!, {
-      writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
-      readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
-    })
-  }
 
   return createdTable
 }

packages/server/src/sdk/app/views/index.ts

@@ -2,7 +2,6 @@ import {
   BBReferenceFieldSubType,
   CalculationType,
   canGroupBy,
-  FeatureFlag,
   FieldType,
   isNumeric,
   PermissionLevel,
@@ -16,7 +15,7 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { context, docIds, features, HTTPError } from "@budibase/backend-core"
+import { context, docIds, HTTPError } from "@budibase/backend-core"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -287,17 +286,12 @@ export async function create(
   await guardViewSchema(tableId, viewRequest)
   const view = await pickApi(tableId).create(tableId, viewRequest)
 
-  const setExplicitPermission = await features.flags.isEnabled(
+  // Set permissions to be the same as the table
-    FeatureFlag.TABLES_DEFAULT_ADMIN
+  const tablePerms = await sdk.permissions.getResourcePerms(tableId)
-  )
+  await sdk.permissions.setPermissions(view.id, {
-  if (setExplicitPermission) {
+    writeRole: tablePerms[PermissionLevel.WRITE].role,
-    // Set permissions to be the same as the table
+    readRole: tablePerms[PermissionLevel.READ].role,
-    const tablePerms = await sdk.permissions.getResourcePerms(tableId)
+  })
-    await sdk.permissions.setPermissions(view.id, {
-      writeRole: tablePerms[PermissionLevel.WRITE].role,
-      readRole: tablePerms[PermissionLevel.READ].role,
-    })
-  }
 
   return view
 }

packages/types/src/sdk/featureFlag.ts

@@ -6,7 +6,6 @@ export enum FeatureFlag {
   AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
   DEFAULT_VALUES = "DEFAULT_VALUES",
   ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",
-  TABLES_DEFAULT_ADMIN = "TABLES_DEFAULT_ADMIN",
   BUDIBASE_AI = "BUDIBASE_AI",
 }