Merge branch 'master' into form-block-field-layout

.github/workflows/deploy-qa.yml

@@ -17,6 +17,6 @@ jobs:
           token: ${{ secrets.GH_ACCESS_TOKEN }}
           client-payload: |-
             {
-              "PAYLOAD_VERSION": "${{ github.sha }}",
+              "VERSION": "${{ github.sha }}",
               "REF_NAME": "${{ github.ref_name}}"
             }

.github/workflows/release-master.yml

@@ -170,9 +170,9 @@ jobs:
         with:
           repository: budibase/budibase-deploys
           event-type: budicloud-qa-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
+          token: ${{ secrets.GH_ACCESS_TOKEN }}
           client-payload: |-
             {
-              "PAYLOAD_VERSION": "${{ github.ref_name }}",
+              "VERSION": "${{ github.ref_name }}",
               "REF_NAME": "${{ github.ref_name}}"
             }

hosting/single/runner.sh

@@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 /bbcouch-runner.sh &
-minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.12.7",
+  "version": "2.12.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/index.ts

@@ -30,7 +30,6 @@ export * as timers from "./timers"
 export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
-export * from "./utils/Duration"
 export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal

packages/backend-core/src/objectStore/utils.ts

@@ -18,8 +18,12 @@ export const ObjectStoreBuckets = {
 }
 
 const bbTmp = join(tmpdir(), ".budibase")
-if (!fs.existsSync(bbTmp)) {
+try {
   fs.mkdirSync(bbTmp)
+} catch (e: any) {
+  if (e.code !== "EEXIST") {
+    throw e
+  }
 }
 
 export function budibaseTempDir() {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -36,7 +36,7 @@ class InMemoryQueue {
    * @param opts This is not used by the in memory queue as there is no real use
    * case when in memory, but is the same API as Bull
    */
-  constructor(name: string, opts?: any) {
+  constructor(name: string, opts = null) {
     this._name = name
     this._opts = opts
     this._messages = []

packages/backend-core/src/queue/queue.ts

@@ -2,18 +2,11 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions } from "bull"
+import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-import { Duration } from "../utils"
 import * as timers from "../timers"
-import * as Redis from "ioredis"
 
-// the queue lock is held for 5 minutes
-const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
-// queue lock is refreshed every 30 seconds
-const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
-// cleanup the queue every 60 seconds
-const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
+const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
 let cleanupInterval: NodeJS.Timeout
 
@@ -28,14 +21,7 @@ export function createQueue<T>(
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
   const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
-  const queueConfig: QueueOptions = {
-    redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
-    settings: {
-      maxStalledCount: 0,
-      lockDuration: QUEUE_LOCK_MS,
-      lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
-    },
-  }
+  const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/backend-core/src/utils/Duration.ts

@@ -1,49 +0,0 @@
-export enum DurationType {
-  MILLISECONDS = "milliseconds",
-  SECONDS = "seconds",
-  MINUTES = "minutes",
-  HOURS = "hours",
-  DAYS = "days",
-}
-
-const conversion: Record<DurationType, number> = {
-  milliseconds: 1,
-  seconds: 1000,
-  minutes: 60 * 1000,
-  hours: 60 * 60 * 1000,
-  days: 24 * 60 * 60 * 1000,
-}
-
-export class Duration {
-  static convert(from: DurationType, to: DurationType, duration: number) {
-    const milliseconds = duration * conversion[from]
-    return milliseconds / conversion[to]
-  }
-
-  static from(from: DurationType, duration: number) {
-    return {
-      to: (to: DurationType) => {
-        return Duration.convert(from, to, duration)
-      },
-      toMs: () => {
-        return Duration.convert(from, DurationType.MILLISECONDS, duration)
-      },
-    }
-  }
-
-  static fromSeconds(duration: number) {
-    return Duration.from(DurationType.SECONDS, duration)
-  }
-
-  static fromMinutes(duration: number) {
-    return Duration.from(DurationType.MINUTES, duration)
-  }
-
-  static fromHours(duration: number) {
-    return Duration.from(DurationType.HOURS, duration)
-  }
-
-  static fromDays(duration: number) {
-    return Duration.from(DurationType.DAYS, duration)
-  }
-}

packages/backend-core/src/utils/index.ts

@@ -1,4 +1,3 @@
 export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
-export * from "./Duration"

packages/backend-core/src/utils/tests/Duration.spec.ts

@@ -1,19 +0,0 @@
-import { Duration, DurationType } from "../Duration"
-
-describe("duration", () => {
-  it("should convert minutes to milliseconds", () => {
-    expect(Duration.fromMinutes(5).toMs()).toBe(300000)
-  })
-
-  it("should convert seconds to milliseconds", () => {
-    expect(Duration.fromSeconds(30).toMs()).toBe(30000)
-  })
-
-  it("should convert days to milliseconds", () => {
-    expect(Duration.fromDays(1).toMs()).toBe(86400000)
-  })
-
-  it("should convert minutes to days", () => {
-    expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
-  })
-})

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -1,27 +1,10 @@
-<script context="module">
-  // We can create a module level cache for all relationship cells to avoid
-  // having to fetch the table definition one time for each cell
-  let primaryDisplayCache = {}
-
-  const getPrimaryDisplayForTableId = async (API, tableId) => {
-    if (primaryDisplayCache[tableId]) {
-      return primaryDisplayCache[tableId]
-    }
-    const definition = await API.fetchTableDefinition(tableId)
-    const primaryDisplay =
-      definition?.primaryDisplay || definition?.schema?.[0]?.name
-    primaryDisplayCache[tableId] = primaryDisplay
-    return primaryDisplay
-  }
-</script>
-
 <script>
   import { getColor } from "../lib/utils"
   import { onMount, getContext } from "svelte"
   import { Icon, Input, ProgressCircle, clickOutside } from "@budibase/bbui"
   import { debounce } from "../../../utils/utils"
 
-  const { API, dispatch } = getContext("grid")
+  const { API, dispatch, cache } = getContext("grid")
 
   export let value
   export let api
@@ -147,7 +130,9 @@
     // Find the primary display for the related table
     if (!primaryDisplay) {
       searching = true
-      primaryDisplay = await getPrimaryDisplayForTableId(API, schema.tableId)
+      primaryDisplay = await cache.actions.getPrimaryDisplayForTableId(
+        schema.tableId
+      )
     }
 
     // Show initial list of results
@@ -195,7 +180,7 @@
   const toggleRow = async row => {
     if (value?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
-      // row to be the the same position if possible
+      // row to be the same position if possible
       if (oneRowOnly) {
         await onChange([])
       } else {
@@ -260,8 +245,7 @@
       class:wrap={editable || contentLines > 1}
       on:wheel={e => (focused ? e.stopPropagation() : null)}
     >
-      {#if Array.isArray(value) && value.length}
-        {#each value as relationship}
+      {#each value || [] as relationship}
         {#if relationship[primaryDisplay] || relationship.primaryDisplay}
           <div class="badge">
             <span
@@ -284,7 +268,6 @@
           </div>
         {/if}
       {/each}
-      {/if}
       {#if editable}
         <div class="add" on:click={open}>
           <Icon name="Add" size="S" />
@@ -320,7 +303,7 @@
         <div class="searching">
           <ProgressCircle size="S" />
         </div>
-      {:else if Array.isArray(searchResults) && searchResults.length}
+      {:else if searchResults?.length}
         <div class="results">
           {#each searchResults as row, idx}
             <div

packages/frontend-core/src/components/grid/stores/cache.js

@@ -0,0 +1,47 @@
+export const createActions = context => {
+  const { API } = context
+
+  // Cache for the primary display columns of different tables.
+  // If we ever need to cache table definitions for other purposes then we can
+  // expand this to be a more generic cache.
+  let primaryDisplayCache = {}
+
+  const resetPrimaryDisplayCache = () => {
+    primaryDisplayCache = {}
+  }
+
+  const getPrimaryDisplayForTableId = async tableId => {
+    // If we've never encountered this tableId before then store a promise that
+    // resolves to the primary display so that subsequent invocations before the
+    // promise completes can reuse this promise
+    if (!primaryDisplayCache[tableId]) {
+      primaryDisplayCache[tableId] = new Promise(resolve => {
+        API.fetchTableDefinition(tableId).then(def => {
+          const display = def?.primaryDisplay || def?.schema?.[0]?.name
+          primaryDisplayCache[tableId] = display
+          resolve(display)
+        })
+      })
+    }
+
+    // We await the result so that we account for both promises and primitives
+    return await primaryDisplayCache[tableId]
+  }
+
+  return {
+    cache: {
+      actions: {
+        getPrimaryDisplayForTableId,
+        resetPrimaryDisplayCache,
+      },
+    },
+  }
+}
+
+export const initialise = context => {
+  const { datasource, cache } = context
+
+  // Wipe the caches whenever the datasource changes to ensure we aren't
+  // storing any stale information
+  datasource.subscribe(cache.actions.resetPrimaryDisplayCache)
+}

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -160,11 +160,6 @@ export const createActions = context => {
     return getAPI()?.actions.canUseColumn(name)
   }
 
-  // Gets the default number of rows for a single page
-  const getFeatures = () => {
-    return getAPI()?.actions.getFeatures()
-  }
-
   return {
     datasource: {
       ...datasource,
@@ -177,7 +172,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js

@@ -35,11 +35,6 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
-  const getFeatures = () => {
-    // We don't support any features
-    return {}
-  }
-
   return {
     nonPlus: {
       actions: {
@@ -50,7 +45,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/table.js

@@ -1,5 +1,4 @@
 import { get } from "svelte/store"
-import TableFetch from "../../../../fetch/TableFetch"
 
 const SuppressErrors = true
 
@@ -46,10 +45,6 @@ export const createActions = context => {
     return $columns.some(col => col.name === name) || $sticky?.name === name
   }
 
-  const getFeatures = () => {
-    return new TableFetch({ API }).determineFeatureFlags()
-  }
-
   return {
     table: {
       actions: {
@@ -60,7 +55,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,5 +1,4 @@
 import { get } from "svelte/store"
-import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
 
 const SuppressErrors = true
 
@@ -46,10 +45,6 @@ export const createActions = context => {
     )
   }
 
-  const getFeatures = () => {
-    return new ViewV2Fetch({ API }).determineFeatureFlags()
-  }
-
   return {
     viewV2: {
       actions: {
@@ -60,7 +55,6 @@ export const createActions = context => {
         getRow,
         isDatasourceValid,
         canUseColumn,
-        getFeatures,
       },
     },
   }

packages/frontend-core/src/components/grid/stores/index.js

@@ -19,6 +19,7 @@ import * as Datasource from "./datasource"
 import * as Table from "./datasources/table"
 import * as ViewV2 from "./datasources/viewV2"
 import * as NonPlus from "./datasources/nonPlus"
+import * as Cache from "./cache"
 
 const DependencyOrderedStores = [
   Sort,
@@ -42,6 +43,7 @@ const DependencyOrderedStores = [
   Clipboard,
   Config,
   Notifications,
+  Cache,
 ]
 
 export const attachStores = context => {

packages/frontend-core/src/components/grid/stores/rows.js

@@ -114,10 +114,6 @@ export const createActions = context => {
     const $allFilters = get(allFilters)
     const $sort = get(sort)
 
-    // Determine how many rows to fetch per page
-    const features = datasource.actions.getFeatures()
-    const limit = features?.supportsPagination ? RowPageSize : null
-
     // Create new fetch model
     const newFetch = fetchData({
       API,
@@ -126,8 +122,12 @@ export const createActions = context => {
         filter: $allFilters,
         sortColumn: $sort.column,
         sortOrder: $sort.order,
-        limit,
+        limit: RowPageSize,
         paginate: true,
+
+        // Disable client side limiting, so that for queries and custom data
+        // sources we don't impose fake row limits. We want all the data.
+        clientSideLimiting: false,
       },
     })
 

packages/frontend-core/src/fetch/DataFetch.js

@@ -43,6 +43,11 @@ export default class DataFetch {
 
       // Pagination config
       paginate: true,
+
+      // Client side feature customisation
+      clientSideSearching: true,
+      clientSideSorting: true,
+      clientSideLimiting: true,
     }
 
     // State of the fetch
@@ -208,24 +213,32 @@ export default class DataFetch {
    * Fetches some filtered, sorted and paginated data
    */
   async getPage() {
-    const { sortColumn, sortOrder, sortType, limit } = this.options
+    const {
+      sortColumn,
+      sortOrder,
+      sortType,
+      limit,
+      clientSideSearching,
+      clientSideSorting,
+      clientSideLimiting,
+    } = this.options
     const { query } = get(this.store)
 
     // Get the actual data
     let { rows, info, hasNextPage, cursor, error } = await this.getData()
 
     // If we don't support searching, do a client search
-    if (!this.features.supportsSearch) {
+    if (!this.features.supportsSearch && clientSideSearching) {
       rows = runLuceneQuery(rows, query)
     }
 
     // If we don't support sorting, do a client-side sort
-    if (!this.features.supportsSort) {
+    if (!this.features.supportsSort && clientSideSorting) {
       rows = luceneSort(rows, sortColumn, sortOrder, sortType)
     }
 
     // If we don't support pagination, do a client-side limit
-    if (!this.features.supportsPagination) {
+    if (!this.features.supportsPagination && clientSideLimiting) {
       rows = luceneLimit(rows, limit)
     }
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -49,7 +49,12 @@ describe.each([
   let table: Table
   let tableId: string
 
-  afterAll(setup.afterAll)
+  afterAll(async () => {
+    if (dsProvider) {
+      await dsProvider.stopContainer()
+    }
+    setup.afterAll()
+  })
 
   beforeAll(async () => {
     await config.init()
@@ -521,20 +526,17 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await config.api.row.patch(table._id!, {
+      const row = await config.api.row.patch(table._id!, {
         _id: existing._id!,
         _rev: existing._rev!,
         tableId: table._id!,
         name: "Updated Name",
       })
 
-      expect((res as any).res.statusMessage).toEqual(
-        `${table.name} updated successfully.`
-      )
-      expect(res.body.name).toEqual("Updated Name")
-      expect(res.body.description).toEqual(existing.description)
+      expect(row.name).toEqual("Updated Name")
+      expect(row.description).toEqual(existing.description)
 
-      const savedRow = await loadRow(res.body._id, table._id!)
+      const savedRow = await loadRow(row._id!, table._id!)
 
       expect(savedRow.body.description).toEqual(existing.description)
       expect(savedRow.body.name).toEqual("Updated Name")

packages/server/src/api/routes/tests/table.spec.ts

@@ -492,6 +492,67 @@ describe("/tables", () => {
       }
     })
 
+    it("should succeed when the row is created from the other side of the relationship", async () => {
+      // We found a bug just after releasing this feature where if the row was created from the
+      // users table, not the table linking to it, the migration would succeed but lose the data.
+      // This happened because the order of the documents in the link was reversed.
+      const table = await config.api.table.create({
+        name: "table",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          "user relationship": {
+            type: FieldType.LINK,
+            fieldName: "test",
+            name: "user relationship",
+            constraints: {
+              type: "array",
+              presence: false,
+            },
+            relationshipType: RelationshipType.MANY_TO_ONE,
+            tableId: InternalTable.USER_METADATA,
+          },
+        },
+      })
+
+      let testRow = await config.api.row.save(table._id!, {})
+
+      await Promise.all(
+        users.map(u =>
+          config.api.row.patch(InternalTable.USER_METADATA, {
+            tableId: InternalTable.USER_METADATA,
+            _rev: u._rev!,
+            _id: u._id!,
+            test: [testRow],
+          })
+        )
+      )
+
+      await config.api.table.migrate(table._id!, {
+        oldColumn: table.schema["user relationship"],
+        newColumn: {
+          name: "user column",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldSubtype.USERS,
+        },
+      })
+
+      const migratedTable = await config.api.table.get(table._id!)
+      expect(migratedTable.schema["user column"]).toBeDefined()
+      expect(migratedTable.schema["user relationship"]).not.toBeDefined()
+
+      const resp = await config.api.row.get(table._id!, testRow._id!)
+      const migratedRow = resp.body as Row
+
+      expect(migratedRow["user column"]).toBeDefined()
+      expect(migratedRow["user relationship"]).not.toBeDefined()
+      expect(migratedRow["user column"]).toHaveLength(3)
+      expect(migratedRow["user column"].map((u: Row) => u._id)).toEqual(
+        expect.arrayContaining(users.map(u => u._id))
+      )
+    })
+
     it("should successfully migrate a many-to-many user relationship to a users column", async () => {
       const table = await config.api.table.create({
         name: "table",

packages/server/src/tests/utilities/api/row.ts

@@ -55,7 +55,13 @@ export class RowAPI extends TestAPI {
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(expectStatus)
+    if (resp.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          resp.status
+        }, body: ${JSON.stringify(resp.body)}`
+      )
+    }
     return resp.body as Row
   }
 
@@ -77,13 +83,20 @@ export class RowAPI extends TestAPI {
     sourceId: string,
     row: PatchRowRequest,
     { expectStatus } = { expectStatus: 200 }
-  ) => {
-    return this.request
+  ): Promise<Row> => {
+    let resp = await this.request
       .patch(`/api/${sourceId}/rows`)
       .send(row)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
-      .expect(expectStatus)
+    if (resp.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          resp.status
+        }, body: ${JSON.stringify(resp.body)}`
+      )
+    }
+    return resp.body as Row
   }
 
   delete = async (

pull_request_template.md

@@ -1,16 +1,12 @@
 ## Description
-
 _Describe the problem or feature in addition to a link to the relevant github issues._
 
-Addresses:
-
+### Addresses:
 - `<Enter the Link to the issue(s) this PR addresses>`
 - ...more if required
 
 ## App Export
-
 - If possible, attach an app export file along with your request template to make QA testing easier, with minimal setup.
 
 ## Screenshots
-
 _If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._

scripts/install-minio.sh

@@ -2,9 +2,9 @@
 if [[ $TARGETARCH == arm* ]] ;
 then
   echo "INSTALLING ARM64 MINIO"
-  wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb
+  wget https://dl.min.io/server/minio/release/linux-arm64/minio
 else
   echo "INSTALLING AMD64 MINIO"
-  wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb
+  wget https://dl.min.io/server/minio/release/linux-amd64/minio
 fi
-dpkg -i minio.deb
+chmod +x minio