Merge branch 'master' into return-unauthorized-instead-of-forbidden

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.15",
+  "version": "2.29.18",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/db/couch/connections.ts

@@ -25,7 +25,10 @@ export const getCouchInfo = (connection?: string) => {
   }
   const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
   let sqlUrl = env.COUCH_DB_SQL_URL
-  if (!sqlUrl && urlInfo.url) {
+  // default for dev
+  if (env.isDev() && !sqlUrl) {
+    sqlUrl = "http://localhost:4006"
+  } else if (!sqlUrl && urlInfo.url) {
     const parsed = new URL(urlInfo.url)
     // attempt to connect on default port
     sqlUrl = urlInfo.url.replace(parsed.port, "4984")

packages/backend-core/src/db/utils.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
-import { getTenantId, getGlobalDBName } from "../context"
+import { getTenantId, getGlobalDBName, isMultiTenant } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
 import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
@@ -213,6 +213,11 @@ export function isSqsEnabledForTenant(): boolean {
     return false
   }
 
+  // single tenant (self host and dev) always enabled if flag set
+  if (!isMultiTenant()) {
+    return true
+  }
+
   // This is to guard against the situation in tests where tests pass because
   // we're not actually using SQS, we're using Lucene and the tests pass due to
   // parity.
@@ -222,5 +227,13 @@ export function isSqsEnabledForTenant(): boolean {
     )
   }
 
+  // Special case to enable all tenants, for testing in QA.
+  if (
+    env.SQS_SEARCH_ENABLE_TENANTS.length === 1 &&
+    env.SQS_SEARCH_ENABLE_TENANTS[0] === "*"
+  ) {
+    return true
+  }
+
   return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId)
 }

packages/backend-core/src/environment.ts

@@ -114,7 +114,7 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
   SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
   SQS_SEARCH_ENABLE_TENANTS:
     process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [],

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -2,6 +2,7 @@ import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { generator, structures } from "../../../tests"
 import RedisWrapper from "../redis"
 import { env } from "../.."
+import { randomUUID } from "crypto"
 
 jest.setTimeout(30000)
 
@@ -52,10 +53,10 @@ describe("redis", () => {
   describe("bulkStore", () => {
     function createRandomObject(
       keyLength: number,
-      valueGenerator: () => any = () => generator.word()
+      valueGenerator: () => any = () => randomUUID()
     ) {
       return generator
-        .unique(() => generator.word(), keyLength)
+        .unique(() => randomUUID(), keyLength)
         .reduce((acc, key) => {
           acc[key] = valueGenerator()
           return acc

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -30,6 +30,16 @@
       return lowerA > lowerB ? 1 : -1
     })
 
+  $: groupedAutomations = filteredAutomations.reduce((acc, auto) => {
+    acc[auto.definition.trigger.event] ??= {
+      icon: auto.definition.trigger.icon,
+      name: (auto.definition.trigger?.name || "").toUpperCase(),
+      entries: [],
+    }
+    acc[auto.definition.trigger.event].entries.push(auto)
+    return acc
+  }, {})
+
   $: showNoResults = searchString && !filteredAutomations.length
 
   onMount(async () => {
@@ -55,16 +65,25 @@
     />
   </div>
   <div class="side-bar-nav">
-    {#each filteredAutomations as automation}
+    {#each Object.values(groupedAutomations || {}) as triggerGroup}
-      <NavItem
+      <div class="nav-group">
-        text={automation.name}
+        <div class="nav-group-header" title={triggerGroup?.name}>
-        selected={automation._id === selectedAutomationId}
+          {triggerGroup?.name}
-        on:click={() => selectAutomation(automation._id)}
+        </div>
-        selectedBy={$userSelectedResourceMap[automation._id]}
+        {#each triggerGroup.entries as automation}
-        disabled={automation.disabled}
+          <NavItem
-      >
+            icon={triggerGroup.icon}
-        <EditAutomationPopover {automation} />
+            iconColor={"var(--spectrum-global-color-gray-900)"}
-      </NavItem>
+            text={automation.name}
+            selected={automation._id === selectedAutomationId}
+            on:click={() => selectAutomation(automation._id)}
+            selectedBy={$userSelectedResourceMap[automation._id]}
+            disabled={automation.disabled}
+          >
+            <EditAutomationPopover {automation} />
+          </NavItem>
+        {/each}
+      </div>
     {/each}
 
     {#if showNoResults}
@@ -82,6 +101,17 @@
 </Modal>
 
 <style>
+  .nav-group {
+    padding-top: var(--spacing-l);
+  }
+  .nav-group-header {
+    color: var(--spectrum-global-color-gray-600);
+    padding: 0px calc(var(--spacing-l) + 4px);
+    padding-bottom: var(--spacing-l);
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
   .side-bar {
     flex: 0 0 260px;
     display: flex;
@@ -104,7 +134,7 @@
     justify-content: flex-start;
     align-items: center;
     gap: var(--spacing-l);
-    padding: 0 var(--spacing-l);
+    padding: 0 calc(var(--spacing-l) + 4px);
   }
   .side-bar-nav {
     flex: 1 1 auto;

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -250,6 +250,7 @@
                   },
                 }))
               },
+              placeholder: false,
               getOptionLabel: type => type.name,
               getOptionValue: type => type.id,
               options: [
@@ -280,14 +281,14 @@
                   tableId: inputData["row"].tableId,
                 },
                 meta: {
-                  fields: inputData["meta"].oldFields || {},
+                  fields: inputData["meta"]?.oldFields || {},
                 },
                 onChange: e => {
                   onChange({
                     oldRow: e.detail.row,
                     meta: {
-                      fields: inputData["meta"].fields,
+                      fields: inputData["meta"]?.fields || {},
-                      oldFields: e.detail.meta.fields,
+                      oldFields: e.detail?.meta?.fields || {},
                     },
                   })
                 },
@@ -304,7 +305,15 @@
               row: inputData["row"],
               meta: inputData["meta"] || {},
               onChange: e => {
-                onChange(e.detail)
+                onChange({
+                  row: e.detail.row,
+                  meta: {
+                    fields: e.detail?.meta?.fields || {},
+                    ...(isTestModal
+                      ? { oldFields: inputData["meta"]?.oldFields || {} }
+                      : {}),
+                  },
+                })
               },
               ...baseProps,
             },
@@ -452,7 +461,6 @@
    */
   const onChange = Utils.sequential(async update => {
     const request = cloneDeep(update)
-
     // Process app trigger updates
     if (isTrigger && !isTestModal) {
       // Row trigger

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -288,29 +288,27 @@
 {/each}
 
 {#if table && schemaFields}
-  {#key editableFields}
+  <div
-    <div
+    class="add-fields-btn"
-      class="add-fields-btn"
+    class:empty={Object.is(editableFields, {})}
-      class:empty={Object.is(editableFields, {})}
+    bind:this={popoverAnchor}
-      bind:this={popoverAnchor}
+  >
-    >
+    <ActionButton
-      <ActionButton
+      icon="Add"
-        icon="Add"
+      fullWidth
-        fullWidth
+      on:click={() => {
-        on:click={() => {
+        customPopover.show()
-          customPopover.show()
+      }}
-        }}
+      disabled={!schemaFields}
-        disabled={!schemaFields}
+      >Add fields
-        >Add fields
+    </ActionButton>
-      </ActionButton>
+  </div>
-    </div>
-  {/key}
 {/if}
 
 <Popover
   align="center"
   bind:this={customPopover}
-  anchor={popoverAnchor}
+  anchor={editableFields ? popoverAnchor : null}
   useAnchorWidth
   maxHeight={300}
   resizable={false}

packages/server/src/api/routes/tests/search.spec.ts

@@ -809,6 +809,20 @@ describe.each([
           },
         }).toContainExactly([{ name: "foo" }, { name: "bar" }])
       })
+
+      it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => {
+        await expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+          oneOf: { name: [] },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+      })
+
+      it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
+        await expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+          oneOf: { name: [] },
+        }).toContainExactly([])
+      })
     })
 
     describe("fuzzy", () => {

packages/server/src/constants/index.ts

@@ -24,16 +24,6 @@ export enum FilterTypes {
   ONE_OF = "oneOf",
 }
 
-export const NoEmptyFilterStrings = [
-  FilterTypes.STRING,
-  FilterTypes.FUZZY,
-  FilterTypes.EQUAL,
-  FilterTypes.NOT_EQUAL,
-  FilterTypes.CONTAINS,
-  FilterTypes.NOT_CONTAINS,
-  FilterTypes.CONTAINS_ANY,
-]
-
 export const CanSwitchTypes = [
   [FieldType.JSON, FieldType.ARRAY],
   [

packages/server/src/environment.ts

@@ -28,7 +28,6 @@ const DEFAULTS = {
   PLUGINS_DIR: "/plugins",
   FORKED_PROCESS_NAME: "main",
   JS_RUNNER_MEMORY_LIMIT: 64,
-  COUCH_DB_SQL_URL: "http://localhost:4006",
 }
 
 const QUERY_THREAD_TIMEOUT =
@@ -44,7 +43,7 @@ const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
   AWS_REGION: process.env.AWS_REGION,

packages/server/src/sdk/app/rows/search.ts

@@ -2,14 +2,12 @@ import {
   EmptyFilterOption,
   Row,
   RowSearchParams,
-  SearchFilters,
   SearchResponse,
   SortOrder,
 } from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
-import { NoEmptyFilterStrings } from "../../../constants"
 import * as sqs from "./search/sqs"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
@@ -32,44 +30,11 @@ function pickApi(tableId: any) {
   return internal
 }
 
-function isEmptyArray(value: any) {
-  return Array.isArray(value) && value.length === 0
-}
-
-// don't do a pure falsy check, as 0 is included
-// https://github.com/Budibase/budibase/issues/10118
-export function removeEmptyFilters(filters: SearchFilters) {
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!filters[filterField]) {
-      continue
-    }
-
-    for (let filterType of Object.keys(filters)) {
-      if (filterType !== filterField) {
-        continue
-      }
-      // don't know which one we're checking, type could be anything
-      const value = filters[filterType] as unknown
-      if (typeof value === "object") {
-        for (let [key, value] of Object.entries(
-          filters[filterType] as object
-        )) {
-          if (value == null || value === "" || isEmptyArray(value)) {
-            // @ts-ignore
-            delete filters[filterField][key]
-          }
-        }
-      }
-    }
-  }
-  return filters
-}
-
 export async function search(
   options: RowSearchParams
 ): Promise<SearchResponse<Row>> {
   const isExternalTable = isExternalTableID(options.tableId)
-  options.query = removeEmptyFilters(options.query || {})
+  options.query = dataFilters.cleanupQuery(options.query || {})
   options.query = dataFilters.fixupFilterArrays(options.query)
   if (
     !dataFilters.hasFilters(options.query) &&

packages/shared-core/src/filters.ts

@@ -106,31 +106,49 @@ export const NoEmptyFilterStrings = [
   OperatorOptions.NotEquals.value,
   OperatorOptions.Contains.value,
   OperatorOptions.NotContains.value,
+  OperatorOptions.ContainsAny.value,
+  OperatorOptions.In.value,
 ] as (keyof SearchQueryFields)[]
 
 /**
  * Removes any fields that contain empty strings that would cause inconsistent
  * behaviour with how backend tables are filtered (no value means no filter).
+ *
+ * don't do a pure falsy check, as 0 is included
+ * https://github.com/Budibase/budibase/issues/10118
  */
-const cleanupQuery = (query: SearchFilters) => {
+export const cleanupQuery = (query: SearchFilters) => {
   if (!query) {
     return query
   }
   for (let filterField of NoEmptyFilterStrings) {
-    const operator = filterField as SearchFilterOperator
+    if (!query[filterField]) {
-    if (!query[operator]) {
       continue
     }
 
-    for (let [key, value] of Object.entries(query[operator]!)) {
+    for (let filterType of Object.keys(query)) {
-      if (value == null || value === "") {
+      if (filterType !== filterField) {
-        delete query[operator]![key]
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = query[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(query[filterType] as object)) {
+          if (value == null || value === "" || isEmptyArray(value)) {
+            // @ts-ignore
+            delete query[filterField][key]
+          }
+        }
       }
     }
   }
   return query
 }
 
+function isEmptyArray(value: any) {
+  return Array.isArray(value) && value.length === 0
+}
+
 /**
  * Removes a numeric prefix on field names designed to give fields uniqueness
  */

packages/worker/src/api/controllers/system/environment.ts

@@ -1,6 +1,6 @@
 import { Ctx, MaintenanceType } from "@budibase/types"
 import env from "../../../environment"
-import { env as coreEnv } from "@budibase/backend-core"
+import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
 
 let sqsAvailable: boolean
@@ -12,7 +12,12 @@ async function isSqsAvailable() {
   }
 
   try {
-    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+    const couchInfo = dbCore.getCouchInfo()
+    if (!couchInfo.sqlUrl) {
+      sqsAvailable = false
+      return false
+    }
+    await nodeFetch(couchInfo.sqlUrl, {
       timeout: 1000,
     })
     sqsAvailable = true